llvm.org GIT mirror llvm / 96172cb
Add Constant Hoisting Pass Retry commit r200022 with a fix for the build bot errors. Constant expressions have (unlike instructions) module scope use lists and therefore may have users in different functions. The fix is to simply ignore these out-of-function uses. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@200034 91177308-0d34-0410-b5e6-96231b3b80d8 Juergen Ributzka 6 years ago
20 changed file(s) with 728 addition(s) and 46 deletion(s). Raw diff Collapse all Expand all
9191 enum TargetCostConstants {
9292 TCC_Free = 0, ///< Expected to fold away in lowering.
9393 TCC_Basic = 1, ///< The cost of a typical 'add' instruction.
94 TCC_Load = 3,
9495 TCC_Expensive = 4 ///< The cost of a 'div' instruction on x86.
9596 };
9697
298299 /// immediate of the specified type.
299300 virtual unsigned getIntImmCost(const APInt &Imm, Type *Ty) const;
300301
302 /// \brief Return the expected cost of materialization for the given integer
303 /// immediate of the specified type for a given instruction. The cost can be
304 /// zero if the immediate can be folded into the specified instruction.
305 virtual unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
306 Type *Ty) const;
307 virtual unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
308 Type *Ty) const;
301309 /// @}
302310
303311 /// \name Vector Target Information
400400 //===--------------------------------------------------------------------===//
401401 // Node creation methods.
402402 //
403 SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false);
404 SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false);
405 SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false);
403 SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false,
404 bool isOpaque = false);
405 SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false,
406 bool isOpaque = false);
407 SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false,
408 bool isOpaque = false);
406409 SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false);
407 SDValue getTargetConstant(uint64_t Val, EVT VT) {
408 return getConstant(Val, VT, true);
409 }
410 SDValue getTargetConstant(const APInt &Val, EVT VT) {
411 return getConstant(Val, VT, true);
412 }
413 SDValue getTargetConstant(const ConstantInt &Val, EVT VT) {
414 return getConstant(Val, VT, true);
410 SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) {
411 return getConstant(Val, VT, true, isOpaque);
412 }
413 SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) {
414 return getConstant(Val, VT, true, isOpaque);
415 }
416 SDValue getTargetConstant(const ConstantInt &Val, EVT VT,
417 bool isOpaque = false) {
418 return getConstant(Val, VT, true, isOpaque);
415419 }
416420 // The forms below that take a double should only be used for simple
417421 // constants that can be exactly represented in VT. No checks are made.
12491249 class ConstantSDNode : public SDNode {
12501250 const ConstantInt *Value;
12511251 friend class SelectionDAG;
1252 ConstantSDNode(bool isTarget, const ConstantInt *val, EVT VT)
1252 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
12531253 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant,
12541254 0, DebugLoc(), getSDVTList(VT)), Value(val) {
1255 SubclassData |= isOpaque;
12551256 }
12561257 public:
12571258
12631264 bool isOne() const { return Value->isOne(); }
12641265 bool isNullValue() const { return Value->isNullValue(); }
12651266 bool isAllOnesValue() const { return Value->isAllOnesValue(); }
1267
1268 bool isOpaque() const { return SubclassData & 1; }
12661269
12671270 static bool classof(const SDNode *N) {
12681271 return N->getOpcode() == ISD::Constant ||
8989 void initializeFlattenCFGPassPass(PassRegistry&);
9090 void initializeStructurizeCFGPass(PassRegistry&);
9191 void initializeCFGViewerPass(PassRegistry&);
92 void initializeConstantHoistingPass(PassRegistry&);
9293 void initializeCodeGenPreparePass(PassRegistry&);
9394 void initializeConstantMergePass(PassRegistry&);
9495 void initializeConstantPropagationPass(PassRegistry&);
128128 (void) llvm::createJumpThreadingPass();
129129 (void) llvm::createUnifyFunctionExitNodesPass();
130130 (void) llvm::createInstCountPass();
131 (void) llvm::createConstantHoistingPass();
131132 (void) llvm::createCodeGenPreparePass();
132133 (void) llvm::createEarlyCSEPass();
133134 (void) llvm::createGVNPass();
311311
312312 //===----------------------------------------------------------------------===//
313313 //
314 // ConstantHoisting - This pass prepares a function for expensive constants.
315 //
316 FunctionPass *createConstantHoistingPass();
317
318 //===----------------------------------------------------------------------===//
319 //
314320 // CodeGenPrepare - This pass prepares a function for instruction selection.
315321 //
316322 FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = 0);
155155
156156 unsigned TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const {
157157 return PrevTTI->getIntImmCost(Imm, Ty);
158 }
159
160 unsigned TargetTransformInfo::getIntImmCost(unsigned Opcode, const APInt &Imm,
161 Type *Ty) const {
162 return PrevTTI->getIntImmCost(Opcode, Imm, Ty);
163 }
164
165 unsigned TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
166 Type *Ty) const {
167 return PrevTTI->getIntImmCost(IID, Imm, Ty);
158168 }
159169
160170 unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
540550 }
541551
542552 unsigned getIntImmCost(const APInt &Imm, Type *Ty) const LLVM_OVERRIDE {
543 return 1;
553 return TCC_Basic;
554 }
555
556 unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
557 Type *Ty) const LLVM_OVERRIDE {
558 return TCC_Free;
559 }
560
561 unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
562 Type *Ty) const LLVM_OVERRIDE {
563 return TCC_Free;
544564 }
545565
546566 unsigned getNumberOfRegisters(bool Vector) const LLVM_OVERRIDE {
6969 cl::desc("Disable Machine Sinking"));
7070 static cl::opt DisableLSR("disable-lsr", cl::Hidden,
7171 cl::desc("Disable Loop Strength Reduction Pass"));
72 static cl::opt DisableConstantHoisting("disable-constant-hoisting",
73 cl::Hidden, cl::desc("Disable ConstantHoisting"));
7274 static cl::opt DisableCGP("disable-cgp", cl::Hidden,
7375 cl::desc("Disable Codegen Prepare"));
7476 static cl::opt DisableCopyProp("disable-copyprop", cl::Hidden,
395397
396398 // Make sure that no unreachable blocks are instruction selected.
397399 addPass(createUnreachableBlockEliminationPass());
400
401 // Prepare expensive constants for SelectionDAG.
402 if (getOptLevel() != CodeGenOpt::None && !DisableConstantHoisting)
403 addPass(createConstantHoistingPass());
398404 }
399405
400406 /// Turn exception handling constructs into something the code generators can
32113211 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
32123212 isa(N0.getOperand(1))) {
32133213 ConstantSDNode *C1 = cast(N0.getOperand(1));
3214 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0)
3214 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0) {
3215 SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1);
3216 if (!COR.getNode())
3217 return SDValue();
32153218 return DAG.getNode(ISD::AND, SDLoc(N), VT,
32163219 DAG.getNode(ISD::OR, SDLoc(N0), VT,
3217 N0.getOperand(0), N1),
3218 DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1));
3220 N0.getOperand(0), N1), COR);
3221 }
32193222 }
32203223 // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
32213224 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
383383 llvm_unreachable("Should only be used on nodes with operands");
384384 default: break; // Normal nodes don't need extra info.
385385 case ISD::TargetConstant:
386 case ISD::Constant:
387 ID.AddPointer(cast(N)->getConstantIntValue());
386 case ISD::Constant: {
387 const ConstantSDNode *C = cast(N);
388 ID.AddPointer(C->getConstantIntValue());
389 ID.AddBoolean(C->isOpaque());
388390 break;
391 }
389392 case ISD::TargetConstantFP:
390393 case ISD::ConstantFP: {
391394 ID.AddPointer(cast(N)->getConstantFPValue());
970973 return getNode(ISD::XOR, DL, VT, Val, NegOne);
971974 }
972975
973 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) {
976 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
974977 EVT EltVT = VT.getScalarType();
975978 assert((EltVT.getSizeInBits() >= 64 ||
976979 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
977980 "getConstant with a uint64_t value that doesn't fit in the type!");
978 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT);
979 }
980
981 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) {
982 return getConstant(*ConstantInt::get(*Context, Val), VT, isT);
983 }
984
985 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
981 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
982 }
983
984 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
985 {
986 return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
987 }
988
989 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
990 bool isO) {
986991 assert(VT.isInteger() && "Cannot create FP integer constant!");
987992
988993 EVT EltVT = VT.getScalarType();
10241029 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
10251030 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
10261031 .trunc(ViaEltSizeInBits),
1027 ViaEltVT, isT));
1032 ViaEltVT, isT, isO));
10281033 }
10291034
10301035 // EltParts is currently in little endian order. If we actually want
10551060 FoldingSetNodeID ID;
10561061 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
10571062 ID.AddPointer(Elt);
1063 ID.AddBoolean(isO);
10581064 void *IP = 0;
10591065 SDNode *N = NULL;
10601066 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
10621068 return SDValue(N, 0);
10631069
10641070 if (!N) {
1065 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT);
1071 N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
10661072 CSEMap.InsertNode(N, IP);
10671073 AllNodes.push_back(N);
10681074 }
27882794
27892795 ConstantSDNode *Scalar1 = dyn_cast(Cst1);
27902796 ConstantSDNode *Scalar2 = dyn_cast(Cst2);
2791 if (Scalar1 && Scalar2) {
2797 if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
2798 return SDValue();
2799
2800 if (Scalar1 && Scalar2)
27922801 // Scalar instruction.
27932802 Inputs.push_back(std::make_pair(Scalar1, Scalar2));
2794 } else {
2803 else {
27952804 // For vectors extract each constant element into Inputs so we can constant
27962805 // fold them individually.
27972806 BuildVectorSDNode *BV1 = dyn_cast(Cst1);
28052814 ConstantSDNode *V1 = dyn_cast(BV1->getOperand(I));
28062815 ConstantSDNode *V2 = dyn_cast(BV2->getOperand(I));
28072816 if (!V1 || !V2) // Not a constant, bail.
2817 return SDValue();
2818
2819 if (V1->isOpaque() || V2->isOpaque())
28082820 return SDValue();
28092821
28102822 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
35603572 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
35613573 }
35623574
3563 // If the "cost" of materializing the integer immediate is 1 or free, then
3564 // it is cost effective to turn the load into the immediate.
3575 // If the "cost" of materializing the integer immediate is less than the cost
3576 // of a load, then it is cost effective to turn the load into the immediate.
35653577 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo();
3566 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2)
3578 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) <
3579 TargetTransformInfo::TCC_Load)
35673580 return DAG.getConstant(Val, VT);
35683581 return SDValue(0, 0);
35693582 }
29442944 if (DestVT != N.getValueType())
29452945 setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
29462946 DestVT, N)); // convert types.
2947 else if(ConstantSDNode *C = dyn_cast(N))
2948 setValue(&I, DAG.getConstant(C->getAPIntValue(), C->getValueType(0),
2949 /*isTarget=*/false, /*isOpaque*/true));
29472950 else
29482951 setValue(&I, N); // noop cast.
29492952 }
8080 case ISD::VALUETYPE: return "ValueType";
8181 case ISD::Register: return "Register";
8282 case ISD::RegisterMask: return "RegisterMask";
83 case ISD::Constant: return "Constant";
83 case ISD::Constant:
84 if (cast(this)->isOpaque())
85 return "OpaqueConstant";
86 return "Constant";
8487 case ISD::ConstantFP: return "ConstantFP";
8588 case ISD::GlobalAddress: return "GlobalAddress";
8689 case ISD::GlobalTLSAddress: return "GlobalTLSAddress";
110113 }
111114
112115 case ISD::BUILD_VECTOR: return "BUILD_VECTOR";
113 case ISD::TargetConstant: return "TargetConstant";
116 case ISD::TargetConstant:
117 if (cast(this)->isOpaque())
118 return "OpaqueTargetConstant";
119 return "TargetConstant";
114120 case ISD::TargetConstantFP: return "TargetConstantFP";
115121 case ISD::TargetGlobalAddress: return "TargetGlobalAddress";
116122 case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress";
14691469 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
14701470 if (C1 == MinVal) return DAG.getConstant(1, VT); // X >= MIN --> true
14711471 // X >= C0 --> X > (C0-1)
1472 return DAG.getSetCC(dl, VT, N0,
1473 DAG.getConstant(C1-1, N1.getValueType()),
1474 (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT);
1472 APInt C = C1-1;
1473 if (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1474 isLegalICmpImmediate(C.getSExtValue())))
1475 return DAG.getSetCC(dl, VT, N0,
1476 DAG.getConstant(C, N1.getValueType()),
1477 (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT);
14751478 }
14761479
14771480 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
14781481 if (C1 == MaxVal) return DAG.getConstant(1, VT); // X <= MAX --> true
14791482 // X <= C0 --> X < (C0+1)
1480 return DAG.getSetCC(dl, VT, N0,
1481 DAG.getConstant(C1+1, N1.getValueType()),
1482 (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT);
1483 APInt C = C1+1;
1484 if (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1485 isLegalICmpImmediate(C.getSExtValue())))
1486 return DAG.getSetCC(dl, VT, N0,
1487 DAG.getConstant(C, N1.getValueType()),
1488 (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT);
14831489 }
14841490
14851491 if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal)
1717 #include "X86.h"
1818 #include "X86TargetMachine.h"
1919 #include "llvm/Analysis/TargetTransformInfo.h"
20 #include "llvm/IR/IntrinsicInst.h"
2021 #include "llvm/Support/Debug.h"
2122 #include "llvm/Target/CostTable.h"
2223 #include "llvm/Target/TargetLowering.h"
105106
106107 virtual unsigned getReductionCost(unsigned Opcode, Type *Ty,
107108 bool IsPairwiseForm) const LLVM_OVERRIDE;
109
110 virtual unsigned getIntImmCost(const APInt &Imm,
111 Type *Ty) const LLVM_OVERRIDE;
112
113 virtual unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
114 Type *Ty) const LLVM_OVERRIDE;
115 virtual unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
116 Type *Ty) const LLVM_OVERRIDE;
108117
109118 /// @}
110119 };
693702 return TargetTransformInfo::getReductionCost(Opcode, ValTy, IsPairwise);
694703 }
695704
705 unsigned X86TTI::getIntImmCost(const APInt &Imm, Type *Ty) const {
706 assert(Ty->isIntegerTy());
707
708 unsigned BitSize = Ty->getPrimitiveSizeInBits();
709 if (BitSize == 0)
710 return ~0U;
711
712 if (Imm.getBitWidth() <= 64 &&
713 (isInt<32>(Imm.getSExtValue()) || isUInt<32>(Imm.getZExtValue())))
714 return TCC_Basic;
715 else
716 return 2 * TCC_Basic;
717 }
718
719 unsigned X86TTI::getIntImmCost(unsigned Opcode, const APInt &Imm,
720 Type *Ty) const {
721 assert(Ty->isIntegerTy());
722
723 unsigned BitSize = Ty->getPrimitiveSizeInBits();
724 if (BitSize == 0)
725 return ~0U;
726
727 switch (Opcode) {
728 case Instruction::Add:
729 case Instruction::Sub:
730 case Instruction::Mul:
731 case Instruction::UDiv:
732 case Instruction::SDiv:
733 case Instruction::URem:
734 case Instruction::SRem:
735 case Instruction::Shl:
736 case Instruction::LShr:
737 case Instruction::AShr:
738 case Instruction::And:
739 case Instruction::Or:
740 case Instruction::Xor:
741 case Instruction::ICmp:
742 if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
743 return TCC_Free;
744 else
745 return X86TTI::getIntImmCost(Imm, Ty);
746 case Instruction::Trunc:
747 case Instruction::ZExt:
748 case Instruction::SExt:
749 case Instruction::IntToPtr:
750 case Instruction::PtrToInt:
751 case Instruction::BitCast:
752 case Instruction::Call:
753 case Instruction::Select:
754 case Instruction::Ret:
755 case Instruction::Load:
756 case Instruction::Store:
757 return X86TTI::getIntImmCost(Imm, Ty);
758 }
759 return TargetTransformInfo::getIntImmCost(Opcode, Imm, Ty);
760 }
761
762 unsigned X86TTI::getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
763 Type *Ty) const {
764 assert(Ty->isIntegerTy());
765
766 unsigned BitSize = Ty->getPrimitiveSizeInBits();
767 if (BitSize == 0)
768 return ~0U;
769
770 switch (IID) {
771 default: return TargetTransformInfo::getIntImmCost(IID, Imm, Ty);
772 case Intrinsic::sadd_with_overflow:
773 case Intrinsic::uadd_with_overflow:
774 case Intrinsic::ssub_with_overflow:
775 case Intrinsic::usub_with_overflow:
776 case Intrinsic::smul_with_overflow:
777 case Intrinsic::umul_with_overflow:
778 if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
779 return TCC_Free;
780 else
781 return X86TTI::getIntImmCost(Imm, Ty);
782 case Intrinsic::experimental_stackmap:
783 case Intrinsic::experimental_patchpoint_void:
784 case Intrinsic::experimental_patchpoint_i64:
785 if (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))
786 return TCC_Free;
787 else
788 return X86TTI::getIntImmCost(Imm, Ty);
789 }
790 }
0 add_llvm_library(LLVMScalarOpts
11 ADCE.cpp
22 CodeGenPrepare.cpp
3 ConstantHoisting.cpp
34 ConstantProp.cpp
45 CorrelatedValuePropagation.cpp
56 DCE.cpp
239239 bool CodeGenPrepare::EliminateFallThrough(Function &F) {
240240 bool Changed = false;
241241 // Scan all of the blocks in the function, except for the entry block.
242 for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ) {
242 for (Function::iterator I = llvm::next(F.begin()), E = F.end(); I != E; ) {
243243 BasicBlock *BB = I++;
244244 // If the destination block has a single pred, then this is a trivial
245245 // edge, just collapse it.
275275 bool CodeGenPrepare::EliminateMostlyEmptyBlocks(Function &F) {
276276 bool MadeChange = false;
277277 // Note that this intentionally skips the entry block.
278 for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ) {
278 for (Function::iterator I = llvm::next(F.begin()), E = F.end(); I != E; ) {
279279 BasicBlock *BB = I++;
280280
281281 // If this block doesn't end with an uncond branch, ignore it.
0 //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass identifies expensive constants to hoist and coalesces them to
10 // better prepare it for SelectionDAG-based code generation. This works around
11 // the limitations of the basic-block-at-a-time approach.
12 //
13 // First it scans all instructions for integer constants and calculates its
14 // cost. If the constant can be folded into the instruction (the cost is
15 // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
16 // consider it expensive and leave it alone. This is the default behavior and
17 // the default implementation of getIntImmCost will always return TCC_Free.
18 //
19 // If the cost is more than TCC_BASIC, then the integer constant can't be folded
20 // into the instruction and it might be beneficial to hoist the constant.
21 // Similar constants are coalesced to reduce register pressure and
22 // materialization code.
23 //
24 // When a constant is hoisted, it is also hidden behind a bitcast to force it to
25 // be live-out of the basic block. Otherwise the constant would be just
26 // duplicated and each basic block would have its own copy in the SelectionDAG.
27 // The SelectionDAG recognizes such constants as opaque and doesn't perform
28 // certain transformations on them, which would create a new expensive constant.
29 //
30 // This optimization is only applied to integer constants in instructions and
31 // simple (this means not nested) constant cast experessions. For example:
32 // %0 = load i64* inttoptr (i64 big_constant to i64*)
33 //===----------------------------------------------------------------------===//
34
35 #define DEBUG_TYPE "consthoist"
36 #include "llvm/Transforms/Scalar.h"
37 #include "llvm/ADT/MapVector.h"
38 #include "llvm/ADT/SmallSet.h"
39 #include "llvm/ADT/Statistic.h"
40 #include "llvm/Analysis/TargetTransformInfo.h"
41 #include "llvm/IR/Constants.h"
42 #include "llvm/IR/Dominators.h"
43 #include "llvm/IR/IntrinsicInst.h"
44 #include "llvm/Pass.h"
45 #include "llvm/Support/CommandLine.h"
46 #include "llvm/Support/Debug.h"
47
48 using namespace llvm;
49
50 STATISTIC(NumConstantsHoisted, "Number of constants hoisted");
51 STATISTIC(NumConstantsRebased, "Number of constants rebased");
52
53
54 namespace {
55 typedef SmallVector ConstantUseListType;
56 struct ConstantCandidate {
57 unsigned CumulativeCost;
58 ConstantUseListType Uses;
59 };
60
61 struct ConstantInfo {
62 ConstantInt *BaseConstant;
63 struct RebasedConstantInfo {
64 ConstantInt *OriginalConstant;
65 Constant *Offset;
66 ConstantUseListType Uses;
67 };
68 typedef SmallVector RebasedConstantListType;
69 RebasedConstantListType RebasedConstants;
70 };
71
72 class ConstantHoisting : public FunctionPass {
73 const TargetTransformInfo *TTI;
74 DominatorTree *DT;
75
76 /// Keeps track of expensive constants found in the function.
77 typedef MapVector ConstantMapType;
78 ConstantMapType ConstantMap;
79
80 /// These are the final constants we decided to hoist.
81 SmallVector Constants;
82 public:
83 static char ID; // Pass identification, replacement for typeid
84 ConstantHoisting() : FunctionPass(ID), TTI(0) {
85 initializeConstantHoistingPass(*PassRegistry::getPassRegistry());
86 }
87
88 bool runOnFunction(Function &F);
89
90 const char *getPassName() const { return "Constant Hoisting"; }
91
92 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
93 AU.setPreservesCFG();
94 AU.addRequired();
95 AU.addRequired();
96 }
97
98 private:
99 void CollectConstant(User *U, unsigned Opcode, Intrinsic::ID IID,
100 ConstantInt *C);
101 void CollectConstants(Instruction *I);
102 void CollectConstants(Function &F);
103 void FindAndMakeBaseConstant(ConstantMapType::iterator S,
104 ConstantMapType::iterator E);
105 void FindBaseConstants();
106 Instruction *FindConstantInsertionPoint(Function &F,
107 const ConstantInfo &CI) const;
108 void EmitBaseConstants(Function &F, User *U, Instruction *Base,
109 Constant *Offset, ConstantInt *OriginalConstant);
110 bool EmitBaseConstants(Function &F);
111 bool OptimizeConstants(Function &F);
112 };
113 }
114
115 char ConstantHoisting::ID = 0;
116 INITIALIZE_PASS_BEGIN(ConstantHoisting, "consthoist", "Constant Hoisting",
117 false, false)
118 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
119 INITIALIZE_AG_DEPENDENCY(TargetTransformInfo)
120 INITIALIZE_PASS_END(ConstantHoisting, "consthoist", "Constant Hoisting",
121 false, false)
122
123 FunctionPass *llvm::createConstantHoistingPass() {
124 return new ConstantHoisting();
125 }
126
127 /// \brief Perform the constant hoisting optimization for the given function.
128 bool ConstantHoisting::runOnFunction(Function &F) {
129 DEBUG(dbgs() << "********** Constant Hoisting **********\n");
130 DEBUG(dbgs() << "********** Function: " << F.getName() << '\n');
131
132 DT = &getAnalysis().getDomTree();
133 TTI = &getAnalysis();
134
135 return OptimizeConstants(F);
136 }
137
138 void ConstantHoisting::CollectConstant(User * U, unsigned Opcode,
139 Intrinsic::ID IID, ConstantInt *C) {
140 unsigned Cost;
141 if (Opcode)
142 Cost = TTI->getIntImmCost(Opcode, C->getValue(), C->getType());
143 else
144 Cost = TTI->getIntImmCost(IID, C->getValue(), C->getType());
145
146 if (Cost > TargetTransformInfo::TCC_Basic) {
147 ConstantCandidate &CC = ConstantMap[C];
148 CC.CumulativeCost += Cost;
149 CC.Uses.push_back(U);
150 }
151 }
152
153 /// \brief Scan the instruction or constant expression for expensive integer
154 /// constants and record them in the constant map.
155 void ConstantHoisting::CollectConstants(Instruction *I) {
156 unsigned Opcode = 0;
157 Intrinsic::ID IID = Intrinsic::not_intrinsic;
158 if (IntrinsicInst *II = dyn_cast(I))
159 IID = II->getIntrinsicID();
160 else
161 Opcode = I->getOpcode();
162
163 // Scan all operands.
164 for (User::op_iterator O = I->op_begin(), E = I->op_end(); O != E; ++O) {
165 if (ConstantInt *C = dyn_cast(O)) {
166 CollectConstant(I, Opcode, IID, C);
167 continue;
168 }
169 if (ConstantExpr *CE = dyn_cast(O)) {
170 // We only handle constant cast expressions.
171 if (!CE->isCast())
172 continue;
173
174 if (ConstantInt *C = dyn_cast(CE->getOperand(0))) {
175 // Ignore the cast expression and use the opcode of the instruction.
176 CollectConstant(CE, Opcode, IID, C);
177 continue;
178 }
179 }
180 }
181 }
182
183 /// \brief Collect all integer constants in the function that cannot be folded
184 /// into an instruction itself.
185 void ConstantHoisting::CollectConstants(Function &F) {
186 for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
187 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
188 CollectConstants(I);
189 }
190
191 /// \brief Compare function for sorting integer constants by type and by value
192 /// within a type in ConstantMaps.
193 static bool
194 ConstantMapLessThan(const std::pair &LHS,
195 const std::pair &RHS) {
196 if (LHS.first->getType() == RHS.first->getType())
197 return LHS.first->getValue().ult(RHS.first->getValue());
198 else
199 return LHS.first->getType()->getBitWidth() <
200 RHS.first->getType()->getBitWidth();
201 }
202
203 /// \brief Find the base constant within the given range and rebase all other
204 /// constants with respect to the base constant.
205 void ConstantHoisting::FindAndMakeBaseConstant(ConstantMapType::iterator S,
206 ConstantMapType::iterator E) {
207 ConstantMapType::iterator MaxCostItr = S;
208 unsigned NumUses = 0;
209 // Use the constant that has the maximum cost as base constant.
210 for (ConstantMapType::iterator I = S; I != E; ++I) {
211 NumUses += I->second.Uses.size();
212 if (I->second.CumulativeCost > MaxCostItr->second.CumulativeCost)
213 MaxCostItr = I;
214 }
215
216 // Don't hoist constants that have only one use.
217 if (NumUses <= 1)
218 return;
219
220 ConstantInfo CI;
221 CI.BaseConstant = MaxCostItr->first;
222 Type *Ty = CI.BaseConstant->getType();
223 // Rebase the constants with respect to the base constant.
224 for (ConstantMapType::iterator I = S; I != E; ++I) {
225 APInt Diff = I->first->getValue() - CI.BaseConstant->getValue();
226 ConstantInfo::RebasedConstantInfo RCI;
227 RCI.OriginalConstant = I->first;
228 RCI.Offset = ConstantInt::get(Ty, Diff);
229 RCI.Uses = llvm_move(I->second.Uses);
230 CI.RebasedConstants.push_back(RCI);
231 }
232 Constants.push_back(CI);
233 }
234
235 /// \brief Finds and combines constants that can be easily rematerialized with
236 /// an add from a common base constant.
237 void ConstantHoisting::FindBaseConstants() {
238 // Sort the constants by value and type. This invalidates the mapping.
239 std::sort(ConstantMap.begin(), ConstantMap.end(), ConstantMapLessThan);
240
241 // Simple linear scan through the sorted constant map for viable merge
242 // candidates.
243 ConstantMapType::iterator MinValItr = ConstantMap.begin();
244 for (ConstantMapType::iterator I = llvm::next(ConstantMap.begin()),
245 E = ConstantMap.end(); I != E; ++I) {
246 if (MinValItr->first->getType() == I->first->getType()) {
247 // Check if the constant is in range of an add with immediate.
248 APInt Diff = I->first->getValue() - MinValItr->first->getValue();
249 if ((Diff.getBitWidth() <= 64) &&
250 TTI->isLegalAddImmediate(Diff.getSExtValue()))
251 continue;
252 }
253 // We either have now a different constant type or the constant is not in
254 // range of an add with immediate anymore.
255 FindAndMakeBaseConstant(MinValItr, I);
256 // Start a new base constant search.
257 MinValItr = I;
258 }
259 // Finalize the last base constant search.
260 FindAndMakeBaseConstant(MinValItr, ConstantMap.end());
261 }
262
263 /// \brief Records the basic block of the instruction or all basic blocks of the
264 /// users of the constant expression.
265 static void CollectBasicBlocks(SmallPtrSet &BBs, Function &F,
266 User *U) {
267 if (Instruction *I = dyn_cast(U))
268 BBs.insert(I->getParent());
269 else if (ConstantExpr *CE = dyn_cast(U))
270 // Find all users of this constant expression.
271 for (Value::use_iterator UU = CE->use_begin(), E = CE->use_end();
272 UU != E; ++UU)
273 // Only record users that are instructions. We don't want to go down a
274 // nested constant expression chain. Also check if the instruction is even
275 // in the current function.
276 if (Instruction *I = dyn_cast(*UU))
277 if(I->getParent()->getParent() == &F)
278 BBs.insert(I->getParent());
279 }
280
281 /// \brief Find an insertion point that dominates all uses.
282 Instruction *ConstantHoisting::
283 FindConstantInsertionPoint(Function &F, const ConstantInfo &CI) const {
284 BasicBlock *Entry = &F.getEntryBlock();
285
286 // Collect all basic blocks.
287 SmallPtrSet BBs;
288 ConstantInfo::RebasedConstantListType::const_iterator RCI, RCE;
289 for (RCI = CI.RebasedConstants.begin(), RCE = CI.RebasedConstants.end();
290 RCI != RCE; ++RCI)
291 for (SmallVectorImpl::const_iterator U = RCI->Uses.begin(),
292 E = RCI->Uses.end(); U != E; ++U)
293 CollectBasicBlocks(BBs, F, *U);
294
295 if (BBs.count(Entry))
296 return Entry->getFirstInsertionPt();
297
298 while (BBs.size() >= 2) {
299 BasicBlock *BB, *BB1, *BB2;
300 BB1 = *BBs.begin();
301 BB2 = *llvm::next(BBs.begin());
302 BB = DT->findNearestCommonDominator(BB1, BB2);
303 if (BB == Entry)
304 return Entry->getFirstInsertionPt();
305 BBs.erase(BB1);
306 BBs.erase(BB2);
307 BBs.insert(BB);
308 }
309 assert((BBs.size() == 1) && "Expected only one element.");
310 return (*BBs.begin())->getFirstInsertionPt();
311 }
312
313 /// \brief Emit materialization code for all rebased constants and update their
314 /// users.
315 void ConstantHoisting::EmitBaseConstants(Function &F, User *U,
316 Instruction *Base, Constant *Offset,
317 ConstantInt *OriginalConstant) {
318 if (Instruction *I = dyn_cast(U)) {
319 Instruction *Mat = Base;
320 if (!Offset->isNullValue()) {
321 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
322 "const_mat", I);
323
324 // Use the same debug location as the instruction we are about to update.
325 Mat->setDebugLoc(I->getDebugLoc());
326
327 DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
328 << " + " << *Offset << ") in BB "
329 << I->getParent()->getName() << '\n' << *Mat << '\n');
330 }
331 DEBUG(dbgs() << "Update: " << *I << '\n');
332 I->replaceUsesOfWith(OriginalConstant, Mat);
333 DEBUG(dbgs() << "To: " << *I << '\n');
334 return;
335 }
336 assert(isa(U) && "Expected a ConstantExpr.");
337 ConstantExpr *CE = cast(U);
338 for (Value::use_iterator UU = CE->use_begin(), E = CE->use_end();
339 UU != E; ++UU) {
340 // We only handel instructions here and won't walk down a ConstantExpr chain
341 // to replace all ConstExpr with instructions.
342 if (Instruction *I = dyn_cast(*UU)) {
343 // Only update constant expressions in the current function.
344 if (I->getParent()->getParent() != &F)
345 continue;
346
347 Instruction *Mat = Base;
348 if (!Offset->isNullValue()) {
349 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
350 "const_mat", I);
351
352 // Use the same debug location as the instruction we are about to
353 // update.
354 Mat->setDebugLoc(I->getDebugLoc());
355
356 DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
357 << " + " << *Offset << ") in BB "
358 << I->getParent()->getName() << '\n' << *Mat << '\n');
359 }
360 Instruction *ICE = CE->getAsInstruction();
361 ICE->replaceUsesOfWith(OriginalConstant, Mat);
362 ICE->insertBefore(I);
363
364 // Use the same debug location as the instruction we are about to update.
365 ICE->setDebugLoc(I->getDebugLoc());
366
367 DEBUG(dbgs() << "Create instruction: " << *ICE << '\n');
368 DEBUG(dbgs() << "Update: " << *I << '\n');
369 I->replaceUsesOfWith(CE, ICE);
370 DEBUG(dbgs() << "To: " << *I << '\n');
371 }
372 }
373 }
374
375 /// \brief Hoist and hide the base constant behind a bitcast and emit
376 /// materialization code for derived constants.
377 bool ConstantHoisting::EmitBaseConstants(Function &F) {
378 bool MadeChange = false;
379 SmallVectorImpl::iterator CI, CE;
380 for (CI = Constants.begin(), CE = Constants.end(); CI != CE; ++CI) {
381 // Hoist and hide the base constant behind a bitcast.
382 Instruction *IP = FindConstantInsertionPoint(F, *CI);
383 IntegerType *Ty = CI->BaseConstant->getType();
384 Instruction *Base = new BitCastInst(CI->BaseConstant, Ty, "const", IP);
385 DEBUG(dbgs() << "Hoist constant (" << *CI->BaseConstant << ") to BB "
386 << IP->getParent()->getName() << '\n');
387 NumConstantsHoisted++;
388
389 // Emit materialization code for all rebased constants.
390 ConstantInfo::RebasedConstantListType::iterator RCI, RCE;
391 for (RCI = CI->RebasedConstants.begin(), RCE = CI->RebasedConstants.end();
392 RCI != RCE; ++RCI) {
393 NumConstantsRebased++;
394 for (SmallVectorImpl::iterator U = RCI->Uses.begin(),
395 E = RCI->Uses.end(); U != E; ++U)
396 EmitBaseConstants(F, *U, Base, RCI->Offset, RCI->OriginalConstant);
397 }
398
399 // Use the same debug location as the last user of the constant.
400 assert(!Base->use_empty() && "The use list is empty!?");
401 assert(isa(Base->use_back()) &&
402 "All uses should be instructions.");
403 Base->setDebugLoc(cast(Base->use_back())->getDebugLoc());
404
405 // Correct for base constant, which we counted above too.
406 NumConstantsRebased--;
407 MadeChange = true;
408 }
409 return MadeChange;
410 }
411
412 /// \brief Optimize expensive integer constants in the given function.
413 bool ConstantHoisting::OptimizeConstants(Function &F) {
414 bool MadeChange = false;
415
416 // Collect all constant candidates.
417 CollectConstants(F);
418
419 // There are no constants to worry about.
420 if (ConstantMap.empty())
421 return MadeChange;
422
423 // Combine constants that can be easily materialized with an add from a common
424 // base constant.
425 FindBaseConstants();
426
427 // Finaly hoist the base constant and emit materializating code for dependent
428 // constants.
429 MadeChange |= EmitBaseConstants(F);
430
431 ConstantMap.clear();
432 Constants.clear();
433
434 return MadeChange;
435 }
2929 initializeADCEPass(Registry);
3030 initializeSampleProfileLoaderPass(Registry);
3131 initializeCodeGenPreparePass(Registry);
32 initializeConstantHoistingPass(Registry);
3233 initializeConstantPropagationPass(Registry);
3334 initializeCorrelatedValuePropagationPass(Registry);
3435 initializeDCEPass(Registry);
3737 define void @t2(i8* nocapture %C) nounwind {
3838 entry:
3939 ; CHECK-LABEL: t2:
40 ; CHECK: ldr [[REG2:r[0-9]+]], [r1, #32]
40 ; CHECK: movw [[REG2:r[0-9]+]], #16716
41 ; CHECK: movt [[REG2:r[0-9]+]], #72
4142 ; CHECK: str [[REG2]], [r0, #32]
4243 ; CHECK: vld1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
4344 ; CHECK: vst1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r0]
7879 ; CHECK: strb [[REG5]], [r0, #6]
7980 ; CHECK: movw [[REG6:r[0-9]+]], #21587
8081 ; CHECK: strh [[REG6]], [r0, #4]
81 ; CHECK: ldr [[REG7:r[0-9]+]],
82 ; CHECK: movw [[REG7:r[0-9]+]], #18500
83 ; CHECK: movt [[REG7:r[0-9]+]], #22866
8284 ; CHECK: str [[REG7]]
8385 tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([7 x i8]* @.str5, i64 0, i64 0), i64 7, i32 1, i1 false)
8486 ret void
0 ; RUN: llc < %s -mtriple=x86_64-darwin -mcpu=corei7 | grep movabsq | count 3
1
2 define i64 @constant_hoisting(i64 %o0, i64 %o1, i64 %o2, i64 %o3, i64 %o4, i64 %o5) {
3 entry:
4 %l0 = and i64 %o0, -281474976710654
5 %c0 = icmp ne i64 %l0, 0
6 br i1 %c0, label %fail, label %bb1
7
8 bb1:
9 %l1 = and i64 %o1, -281474976710654
10 %c1 = icmp ne i64 %l1, 0
11 br i1 %c1, label %fail, label %bb2
12
13 bb2:
14 %l2 = and i64 %o2, -281474976710654
15 %c2 = icmp ne i64 %l2, 0
16 br i1 %c2, label %fail, label %bb3
17
18 bb3:
19 %l3 = and i64 %o3, -281474976710654
20 %c3 = icmp ne i64 %l3, 0
21 br i1 %c3, label %fail, label %bb4
22
23 bb4:
24 %l4 = and i64 %o4, -281474976710653
25 %c4 = icmp ne i64 %l4, 0
26 br i1 %c4, label %fail, label %bb5
27
28 bb5:
29 %l5 = and i64 %o5, -281474976710652
30 %c5 = icmp ne i64 %l5, 0
31 br i1 %c5, label %fail, label %bb6
32
33 bb6:
34 ret i64 %l5
35
36 fail:
37 ret i64 -1
38 }
39
40 define void @constant_expressions() {
41 entry:
42 %0 = load i64* inttoptr (i64 add (i64 51250129900, i64 0) to i64*)
43 %1 = load i64* inttoptr (i64 add (i64 51250129900, i64 8) to i64*)
44 %2 = load i64* inttoptr (i64 add (i64 51250129900, i64 16) to i64*)
45 %3 = load i64* inttoptr (i64 add (i64 51250129900, i64 24) to i64*)
46 %4 = add i64 %0, %1
47 %5 = add i64 %2, %3
48 %6 = add i64 %4, %5
49 store i64 %6, i64* inttoptr (i64 add (i64 51250129900, i64 0) to i64*)
50 ret void
51 }
52
53
54 define void @constant_expressions2() {
55 entry:
56 %0 = load i64* inttoptr (i64 51250129900 to i64*)
57 %1 = load i64* inttoptr (i64 51250129908 to i64*)
58 %2 = load i64* inttoptr (i64 51250129916 to i64*)
59 %3 = load i64* inttoptr (i64 51250129924 to i64*)
60 %4 = add i64 %0, %1
61 %5 = add i64 %2, %3
62 %6 = add i64 %4, %5
63 store i64 %6, i64* inttoptr (i64 51250129900 to i64*)
64 ret void
65 }
66