llvm.org GIT mirror llvm / dc6f9b9
Revert "Add Constant Hoisting Pass" This reverts commit r200022 to unbreak the build bots. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@200024 91177308-0d34-0410-b5e6-96231b3b80d8 Juergen Ributzka 6 years ago
20 changed file(s) with 47 addition(s) and 708 deletion(s). Raw diff Collapse all Expand all
9191 enum TargetCostConstants {
9292 TCC_Free = 0, ///< Expected to fold away in lowering.
9393 TCC_Basic = 1, ///< The cost of a typical 'add' instruction.
94 TCC_Load = 3,
9594 TCC_Expensive = 4 ///< The cost of a 'div' instruction on x86.
9695 };
9796
299298 /// immediate of the specified type.
300299 virtual unsigned getIntImmCost(const APInt &Imm, Type *Ty) const;
301300
302 /// \brief Return the expected cost of materialization for the given integer
303 /// immediate of the specified type for a given instruction. The cost can be
304 /// zero if the immediate can be folded into the specified instruction.
305 virtual unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
306 Type *Ty) const;
307 virtual unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
308 Type *Ty) const;
309301 /// @}
310302
311303 /// \name Vector Target Information
400400 //===--------------------------------------------------------------------===//
401401 // Node creation methods.
402402 //
403 SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false,
404 bool isOpaque = false);
405 SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false,
406 bool isOpaque = false);
407 SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false,
408 bool isOpaque = false);
403 SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false);
404 SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false);
405 SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false);
409406 SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false);
410 SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) {
411 return getConstant(Val, VT, true, isOpaque);
412 }
413 SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) {
414 return getConstant(Val, VT, true, isOpaque);
415 }
416 SDValue getTargetConstant(const ConstantInt &Val, EVT VT,
417 bool isOpaque = false) {
418 return getConstant(Val, VT, true, isOpaque);
407 SDValue getTargetConstant(uint64_t Val, EVT VT) {
408 return getConstant(Val, VT, true);
409 }
410 SDValue getTargetConstant(const APInt &Val, EVT VT) {
411 return getConstant(Val, VT, true);
412 }
413 SDValue getTargetConstant(const ConstantInt &Val, EVT VT) {
414 return getConstant(Val, VT, true);
419415 }
420416 // The forms below that take a double should only be used for simple
421417 // constants that can be exactly represented in VT. No checks are made.
12491249 class ConstantSDNode : public SDNode {
12501250 const ConstantInt *Value;
12511251 friend class SelectionDAG;
1252 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1252 ConstantSDNode(bool isTarget, const ConstantInt *val, EVT VT)
12531253 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant,
12541254 0, DebugLoc(), getSDVTList(VT)), Value(val) {
1255 SubclassData |= isOpaque;
12561255 }
12571256 public:
12581257
12641263 bool isOne() const { return Value->isOne(); }
12651264 bool isNullValue() const { return Value->isNullValue(); }
12661265 bool isAllOnesValue() const { return Value->isAllOnesValue(); }
1267
1268 bool isOpaque() const { return SubclassData & 1; }
12691266
12701267 static bool classof(const SDNode *N) {
12711268 return N->getOpcode() == ISD::Constant ||
8989 void initializeFlattenCFGPassPass(PassRegistry&);
9090 void initializeStructurizeCFGPass(PassRegistry&);
9191 void initializeCFGViewerPass(PassRegistry&);
92 void initializeConstantHoistingPass(PassRegistry&);
9392 void initializeCodeGenPreparePass(PassRegistry&);
9493 void initializeConstantMergePass(PassRegistry&);
9594 void initializeConstantPropagationPass(PassRegistry&);
128128 (void) llvm::createJumpThreadingPass();
129129 (void) llvm::createUnifyFunctionExitNodesPass();
130130 (void) llvm::createInstCountPass();
131 (void) llvm::createConstantHoistingPass();
132131 (void) llvm::createCodeGenPreparePass();
133132 (void) llvm::createEarlyCSEPass();
134133 (void) llvm::createGVNPass();
311311
312312 //===----------------------------------------------------------------------===//
313313 //
314 // ConstantHoisting - This pass prepares a function for expensive constants.
315 //
316 FunctionPass *createConstantHoistingPass();
317
318 //===----------------------------------------------------------------------===//
319 //
320314 // CodeGenPrepare - This pass prepares a function for instruction selection.
321315 //
322316 FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = 0);
155155
156156 unsigned TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const {
157157 return PrevTTI->getIntImmCost(Imm, Ty);
158 }
159
160 unsigned TargetTransformInfo::getIntImmCost(unsigned Opcode, const APInt &Imm,
161 Type *Ty) const {
162 return PrevTTI->getIntImmCost(Opcode, Imm, Ty);
163 }
164
165 unsigned TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
166 Type *Ty) const {
167 return PrevTTI->getIntImmCost(IID, Imm, Ty);
168158 }
169159
170160 unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
550540 }
551541
552542 unsigned getIntImmCost(const APInt &Imm, Type *Ty) const LLVM_OVERRIDE {
553 return TCC_Basic;
554 }
555
556 unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
557 Type *Ty) const LLVM_OVERRIDE {
558 return TCC_Free;
559 }
560
561 unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
562 Type *Ty) const LLVM_OVERRIDE {
563 return TCC_Free;
543 return 1;
564544 }
565545
566546 unsigned getNumberOfRegisters(bool Vector) const LLVM_OVERRIDE {
6969 cl::desc("Disable Machine Sinking"));
7070 static cl::opt DisableLSR("disable-lsr", cl::Hidden,
7171 cl::desc("Disable Loop Strength Reduction Pass"));
72 static cl::opt DisableConstantHoisting("disable-constant-hoisting",
73 cl::Hidden, cl::desc("Disable ConstantHoisting"));
7472 static cl::opt DisableCGP("disable-cgp", cl::Hidden,
7573 cl::desc("Disable Codegen Prepare"));
7674 static cl::opt DisableCopyProp("disable-copyprop", cl::Hidden,
397395
398396 // Make sure that no unreachable blocks are instruction selected.
399397 addPass(createUnreachableBlockEliminationPass());
400
401 // Prepare expensive constants for SelectionDAG.
402 if (getOptLevel() != CodeGenOpt::None && !DisableConstantHoisting)
403 addPass(createConstantHoistingPass());
404398 }
405399
406400 /// Turn exception handling constructs into something the code generators can
32113211 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
32123212 isa(N0.getOperand(1))) {
32133213 ConstantSDNode *C1 = cast(N0.getOperand(1));
3214 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0) {
3215 SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1);
3216 if (!COR.getNode())
3217 return SDValue();
3214 if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0)
32183215 return DAG.getNode(ISD::AND, SDLoc(N), VT,
32193216 DAG.getNode(ISD::OR, SDLoc(N0), VT,
3220 N0.getOperand(0), N1), COR);
3221 }
3217 N0.getOperand(0), N1),
3218 DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1));
32223219 }
32233220 // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
32243221 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
383383 llvm_unreachable("Should only be used on nodes with operands");
384384 default: break; // Normal nodes don't need extra info.
385385 case ISD::TargetConstant:
386 case ISD::Constant: {
387 const ConstantSDNode *C = cast(N);
388 ID.AddPointer(C->getConstantIntValue());
389 ID.AddBoolean(C->isOpaque());
390 break;
391 }
386 case ISD::Constant:
387 ID.AddPointer(cast(N)->getConstantIntValue());
388 break;
392389 case ISD::TargetConstantFP:
393390 case ISD::ConstantFP: {
394391 ID.AddPointer(cast(N)->getConstantFPValue());
973970 return getNode(ISD::XOR, DL, VT, Val, NegOne);
974971 }
975972
976 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
973 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) {
977974 EVT EltVT = VT.getScalarType();
978975 assert((EltVT.getSizeInBits() >= 64 ||
979976 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
980977 "getConstant with a uint64_t value that doesn't fit in the type!");
981 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
982 }
983
984 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
985 {
986 return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
987 }
988
989 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
990 bool isO) {
978 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT);
979 }
980
981 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) {
982 return getConstant(*ConstantInt::get(*Context, Val), VT, isT);
983 }
984
985 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
991986 assert(VT.isInteger() && "Cannot create FP integer constant!");
992987
993988 EVT EltVT = VT.getScalarType();
10291024 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
10301025 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
10311026 .trunc(ViaEltSizeInBits),
1032 ViaEltVT, isT, isO));
1027 ViaEltVT, isT));
10331028 }
10341029
10351030 // EltParts is currently in little endian order. If we actually want
10601055 FoldingSetNodeID ID;
10611056 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
10621057 ID.AddPointer(Elt);
1063 ID.AddBoolean(isO);
10641058 void *IP = 0;
10651059 SDNode *N = NULL;
10661060 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
10681062 return SDValue(N, 0);
10691063
10701064 if (!N) {
1071 N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
1065 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT);
10721066 CSEMap.InsertNode(N, IP);
10731067 AllNodes.push_back(N);
10741068 }
27942788
27952789 ConstantSDNode *Scalar1 = dyn_cast(Cst1);
27962790 ConstantSDNode *Scalar2 = dyn_cast(Cst2);
2797 if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
2798 return SDValue();
2799
2800 if (Scalar1 && Scalar2)
2791 if (Scalar1 && Scalar2) {
28012792 // Scalar instruction.
28022793 Inputs.push_back(std::make_pair(Scalar1, Scalar2));
2803 else {
2794 } else {
28042795 // For vectors extract each constant element into Inputs so we can constant
28052796 // fold them individually.
28062797 BuildVectorSDNode *BV1 = dyn_cast(Cst1);
28142805 ConstantSDNode *V1 = dyn_cast(BV1->getOperand(I));
28152806 ConstantSDNode *V2 = dyn_cast(BV2->getOperand(I));
28162807 if (!V1 || !V2) // Not a constant, bail.
2817 return SDValue();
2818
2819 if (V1->isOpaque() || V2->isOpaque())
28202808 return SDValue();
28212809
28222810 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
35723560 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
35733561 }
35743562
3575 // If the "cost" of materializing the integer immediate is less than the cost
3576 // of a load, then it is cost effective to turn the load into the immediate.
3563 // If the "cost" of materializing the integer immediate is 1 or free, then
3564 // it is cost effective to turn the load into the immediate.
35773565 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo();
3578 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) <
3579 TargetTransformInfo::TCC_Load)
3566 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2)
35803567 return DAG.getConstant(Val, VT);
35813568 return SDValue(0, 0);
35823569 }
29442944 if (DestVT != N.getValueType())
29452945 setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
29462946 DestVT, N)); // convert types.
2947 else if(ConstantSDNode *C = dyn_cast(N))
2948 setValue(&I, DAG.getConstant(C->getAPIntValue(), C->getValueType(0),
2949 /*isTarget=*/false, /*isOpaque*/true));
29502947 else
29512948 setValue(&I, N); // noop cast.
29522949 }
8080 case ISD::VALUETYPE: return "ValueType";
8181 case ISD::Register: return "Register";
8282 case ISD::RegisterMask: return "RegisterMask";
83 case ISD::Constant:
84 if (cast(this)->isOpaque())
85 return "OpaqueConstant";
86 return "Constant";
83 case ISD::Constant: return "Constant";
8784 case ISD::ConstantFP: return "ConstantFP";
8885 case ISD::GlobalAddress: return "GlobalAddress";
8986 case ISD::GlobalTLSAddress: return "GlobalTLSAddress";
113110 }
114111
115112 case ISD::BUILD_VECTOR: return "BUILD_VECTOR";
116 case ISD::TargetConstant:
117 if (cast(this)->isOpaque())
118 return "OpaqueTargetConstant";
119 return "TargetConstant";
113 case ISD::TargetConstant: return "TargetConstant";
120114 case ISD::TargetConstantFP: return "TargetConstantFP";
121115 case ISD::TargetGlobalAddress: return "TargetGlobalAddress";
122116 case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress";
14691469 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
14701470 if (C1 == MinVal) return DAG.getConstant(1, VT); // X >= MIN --> true
14711471 // X >= C0 --> X > (C0-1)
1472 APInt C = C1-1;
1473 if (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1474 isLegalICmpImmediate(C.getSExtValue())))
1475 return DAG.getSetCC(dl, VT, N0,
1476 DAG.getConstant(C, N1.getValueType()),
1477 (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT);
1472 return DAG.getSetCC(dl, VT, N0,
1473 DAG.getConstant(C1-1, N1.getValueType()),
1474 (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT);
14781475 }
14791476
14801477 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
14811478 if (C1 == MaxVal) return DAG.getConstant(1, VT); // X <= MAX --> true
14821479 // X <= C0 --> X < (C0+1)
1483 APInt C = C1+1;
1484 if (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1485 isLegalICmpImmediate(C.getSExtValue())))
1486 return DAG.getSetCC(dl, VT, N0,
1487 DAG.getConstant(C, N1.getValueType()),
1488 (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT);
1480 return DAG.getSetCC(dl, VT, N0,
1481 DAG.getConstant(C1+1, N1.getValueType()),
1482 (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT);
14891483 }
14901484
14911485 if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal)
1717 #include "X86.h"
1818 #include "X86TargetMachine.h"
1919 #include "llvm/Analysis/TargetTransformInfo.h"
20 #include "llvm/IR/IntrinsicInst.h"
2120 #include "llvm/Support/Debug.h"
2221 #include "llvm/Target/CostTable.h"
2322 #include "llvm/Target/TargetLowering.h"
106105
107106 virtual unsigned getReductionCost(unsigned Opcode, Type *Ty,
108107 bool IsPairwiseForm) const LLVM_OVERRIDE;
109
110 virtual unsigned getIntImmCost(const APInt &Imm,
111 Type *Ty) const LLVM_OVERRIDE;
112
113 virtual unsigned getIntImmCost(unsigned Opcode, const APInt &Imm,
114 Type *Ty) const LLVM_OVERRIDE;
115 virtual unsigned getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
116 Type *Ty) const LLVM_OVERRIDE;
117108
118109 /// @}
119110 };
702693 return TargetTransformInfo::getReductionCost(Opcode, ValTy, IsPairwise);
703694 }
704695
705 unsigned X86TTI::getIntImmCost(const APInt &Imm, Type *Ty) const {
706 assert(Ty->isIntegerTy());
707
708 unsigned BitSize = Ty->getPrimitiveSizeInBits();
709 if (BitSize == 0)
710 return ~0U;
711
712 if (Imm.getBitWidth() <= 64 &&
713 (isInt<32>(Imm.getSExtValue()) || isUInt<32>(Imm.getZExtValue())))
714 return TCC_Basic;
715 else
716 return 2 * TCC_Basic;
717 }
718
719 unsigned X86TTI::getIntImmCost(unsigned Opcode, const APInt &Imm,
720 Type *Ty) const {
721 assert(Ty->isIntegerTy());
722
723 unsigned BitSize = Ty->getPrimitiveSizeInBits();
724 if (BitSize == 0)
725 return ~0U;
726
727 switch (Opcode) {
728 case Instruction::Add:
729 case Instruction::Sub:
730 case Instruction::Mul:
731 case Instruction::UDiv:
732 case Instruction::SDiv:
733 case Instruction::URem:
734 case Instruction::SRem:
735 case Instruction::Shl:
736 case Instruction::LShr:
737 case Instruction::AShr:
738 case Instruction::And:
739 case Instruction::Or:
740 case Instruction::Xor:
741 case Instruction::ICmp:
742 if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
743 return TCC_Free;
744 else
745 return X86TTI::getIntImmCost(Imm, Ty);
746 case Instruction::Trunc:
747 case Instruction::ZExt:
748 case Instruction::SExt:
749 case Instruction::IntToPtr:
750 case Instruction::PtrToInt:
751 case Instruction::BitCast:
752 case Instruction::Call:
753 case Instruction::Select:
754 case Instruction::Ret:
755 case Instruction::Load:
756 case Instruction::Store:
757 return X86TTI::getIntImmCost(Imm, Ty);
758 }
759 return TargetTransformInfo::getIntImmCost(Opcode, Imm, Ty);
760 }
761
762 unsigned X86TTI::getIntImmCost(Intrinsic::ID IID, const APInt &Imm,
763 Type *Ty) const {
764 assert(Ty->isIntegerTy());
765
766 unsigned BitSize = Ty->getPrimitiveSizeInBits();
767 if (BitSize == 0)
768 return ~0U;
769
770 switch (IID) {
771 default: return TargetTransformInfo::getIntImmCost(IID, Imm, Ty);
772 case Intrinsic::sadd_with_overflow:
773 case Intrinsic::uadd_with_overflow:
774 case Intrinsic::ssub_with_overflow:
775 case Intrinsic::usub_with_overflow:
776 case Intrinsic::smul_with_overflow:
777 case Intrinsic::umul_with_overflow:
778 if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
779 return TCC_Free;
780 else
781 return X86TTI::getIntImmCost(Imm, Ty);
782 case Intrinsic::experimental_stackmap:
783 case Intrinsic::experimental_patchpoint_void:
784 case Intrinsic::experimental_patchpoint_i64:
785 if (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))
786 return TCC_Free;
787 else
788 return X86TTI::getIntImmCost(Imm, Ty);
789 }
790 }
0 add_llvm_library(LLVMScalarOpts
11 ADCE.cpp
22 CodeGenPrepare.cpp
3 ConstantHoisting.cpp
43 ConstantProp.cpp
54 CorrelatedValuePropagation.cpp
65 DCE.cpp
239239 bool CodeGenPrepare::EliminateFallThrough(Function &F) {
240240 bool Changed = false;
241241 // Scan all of the blocks in the function, except for the entry block.
242 for (Function::iterator I = llvm::next(F.begin()), E = F.end(); I != E; ) {
242 for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ) {
243243 BasicBlock *BB = I++;
244244 // If the destination block has a single pred, then this is a trivial
245245 // edge, just collapse it.
275275 bool CodeGenPrepare::EliminateMostlyEmptyBlocks(Function &F) {
276276 bool MadeChange = false;
277277 // Note that this intentionally skips the entry block.
278 for (Function::iterator I = llvm::next(F.begin()), E = F.end(); I != E; ) {
278 for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ) {
279279 BasicBlock *BB = I++;
280280
281281 // If this block doesn't end with an uncond branch, ignore it.
+0
-429
lib/Transforms/Scalar/ConstantHoisting.cpp less more
None //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass identifies expensive constants to hoist and coalesces them to
10 // better prepare it for SelectionDAG-based code generation. This works around
11 // the limitations of the basic-block-at-a-time approach.
12 //
13 // First it scans all instructions for integer constants and calculates its
14 // cost. If the constant can be folded into the instruction (the cost is
15 // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
16 // consider it expensive and leave it alone. This is the default behavior and
17 // the default implementation of getIntImmCost will always return TCC_Free.
18 //
19 // If the cost is more than TCC_BASIC, then the integer constant can't be folded
20 // into the instruction and it might be beneficial to hoist the constant.
21 // Similar constants are coalesced to reduce register pressure and
22 // materialization code.
23 //
24 // When a constant is hoisted, it is also hidden behind a bitcast to force it to
25 // be live-out of the basic block. Otherwise the constant would be just
26 // duplicated and each basic block would have its own copy in the SelectionDAG.
27 // The SelectionDAG recognizes such constants as opaque and doesn't perform
28 // certain transformations on them, which would create a new expensive constant.
29 //
30 // This optimization is only applied to integer constants in instructions and
31 // simple (this means not nested) constant cast experessions. For example:
32 // %0 = load i64* inttoptr (i64 big_constant to i64*)
33 //===----------------------------------------------------------------------===//
34
35 #define DEBUG_TYPE "consthoist"
36 #include "llvm/Transforms/Scalar.h"
37 #include "llvm/ADT/MapVector.h"
38 #include "llvm/ADT/SmallSet.h"
39 #include "llvm/ADT/Statistic.h"
40 #include "llvm/Analysis/TargetTransformInfo.h"
41 #include "llvm/IR/Constants.h"
42 #include "llvm/IR/Dominators.h"
43 #include "llvm/IR/IntrinsicInst.h"
44 #include "llvm/Pass.h"
45 #include "llvm/Support/CommandLine.h"
46 #include "llvm/Support/Debug.h"
47
48 using namespace llvm;
49
50 STATISTIC(NumConstantsHoisted, "Number of constants hoisted");
51 STATISTIC(NumConstantsRebased, "Number of constants rebased");
52
53
54 namespace {
55 typedef SmallVector ConstantUseListType;
56 struct ConstantCandidate {
57 unsigned CumulativeCost;
58 ConstantUseListType Uses;
59 };
60
61 struct ConstantInfo {
62 ConstantInt *BaseConstant;
63 struct RebasedConstantInfo {
64 ConstantInt *OriginalConstant;
65 Constant *Offset;
66 ConstantUseListType Uses;
67 };
68 typedef SmallVector RebasedConstantListType;
69 RebasedConstantListType RebasedConstants;
70 };
71
72 class ConstantHoisting : public FunctionPass {
73 const TargetTransformInfo *TTI;
74 DominatorTree *DT;
75
76 /// Keeps track of expensive constants found in the function.
77 typedef MapVector ConstantMapType;
78 ConstantMapType ConstantMap;
79
80 /// These are the final constants we decided to hoist.
81 SmallVector Constants;
82 public:
83 static char ID; // Pass identification, replacement for typeid
84 ConstantHoisting() : FunctionPass(ID), TTI(0) {
85 initializeConstantHoistingPass(*PassRegistry::getPassRegistry());
86 }
87
88 bool runOnFunction(Function &F);
89
90 const char *getPassName() const { return "Constant Hoisting"; }
91
92 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
93 AU.setPreservesCFG();
94 AU.addRequired();
95 AU.addRequired();
96 }
97
98 private:
99 void CollectConstant(User *U, unsigned Opcode, Intrinsic::ID IID,
100 ConstantInt *C);
101 void CollectConstants(Instruction *I);
102 void CollectConstants(Function &F);
103 void FindAndMakeBaseConstant(ConstantMapType::iterator S,
104 ConstantMapType::iterator E);
105 void FindBaseConstants();
106 Instruction *FindConstantInsertionPoint(Function &F,
107 const ConstantInfo &CI) const;
108 void EmitBaseConstants(Function &F, User *U, Instruction *Base,
109 Constant *Offset, ConstantInt *OriginalConstant);
110 bool EmitBaseConstants(Function &F);
111 bool OptimizeConstants(Function &F);
112 };
113 }
114
115 char ConstantHoisting::ID = 0;
116 INITIALIZE_PASS_BEGIN(ConstantHoisting, "consthoist", "Constant Hoisting",
117 false, false)
118 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
119 INITIALIZE_AG_DEPENDENCY(TargetTransformInfo)
120 INITIALIZE_PASS_END(ConstantHoisting, "consthoist", "Constant Hoisting",
121 false, false)
122
123 FunctionPass *llvm::createConstantHoistingPass() {
124 return new ConstantHoisting();
125 }
126
127 /// \brief Perform the constant hoisting optimization for the given function.
128 bool ConstantHoisting::runOnFunction(Function &F) {
129 DEBUG(dbgs() << "********** Constant Hoisting **********\n");
130 DEBUG(dbgs() << "********** Function: " << F.getName() << '\n');
131
132 DT = &getAnalysis().getDomTree();
133 TTI = &getAnalysis();
134
135 return OptimizeConstants(F);
136 }
137
138 void ConstantHoisting::CollectConstant(User * U, unsigned Opcode,
139 Intrinsic::ID IID, ConstantInt *C) {
140 unsigned Cost;
141 if (Opcode)
142 Cost = TTI->getIntImmCost(Opcode, C->getValue(), C->getType());
143 else
144 Cost = TTI->getIntImmCost(IID, C->getValue(), C->getType());
145
146 if (Cost > TargetTransformInfo::TCC_Basic) {
147 ConstantCandidate &CC = ConstantMap[C];
148 CC.CumulativeCost += Cost;
149 CC.Uses.push_back(U);
150 }
151 }
152
153 /// \brief Scan the instruction or constant expression for expensive integer
154 /// constants and record them in the constant map.
155 void ConstantHoisting::CollectConstants(Instruction *I) {
156 unsigned Opcode = 0;
157 Intrinsic::ID IID = Intrinsic::not_intrinsic;
158 if (IntrinsicInst *II = dyn_cast(I))
159 IID = II->getIntrinsicID();
160 else
161 Opcode = I->getOpcode();
162
163 // Scan all operands.
164 for (User::op_iterator O = I->op_begin(), E = I->op_end(); O != E; ++O) {
165 if (ConstantInt *C = dyn_cast(O)) {
166 CollectConstant(I, Opcode, IID, C);
167 continue;
168 }
169 if (ConstantExpr *CE = dyn_cast(O)) {
170 // We only handle constant cast expressions.
171 if (!CE->isCast())
172 continue;
173
174 if (ConstantInt *C = dyn_cast(CE->getOperand(0))) {
175 // Ignore the cast expression and use the opcode of the instruction.
176 CollectConstant(CE, Opcode, IID, C);
177 continue;
178 }
179 }
180 }
181 }
182
183 /// \brief Collect all integer constants in the function that cannot be folded
184 /// into an instruction itself.
185 void ConstantHoisting::CollectConstants(Function &F) {
186 for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
187 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
188 CollectConstants(I);
189 }
190
191 /// \brief Compare function for sorting integer constants by type and by value
192 /// within a type in ConstantMaps.
193 static bool
194 ConstantMapLessThan(const std::pair &LHS,
195 const std::pair &RHS) {
196 if (LHS.first->getType() == RHS.first->getType())
197 return LHS.first->getValue().ult(RHS.first->getValue());
198 else
199 return LHS.first->getType()->getBitWidth() <
200 RHS.first->getType()->getBitWidth();
201 }
202
203 /// \brief Find the base constant within the given range and rebase all other
204 /// constants with respect to the base constant.
205 void ConstantHoisting::FindAndMakeBaseConstant(ConstantMapType::iterator S,
206 ConstantMapType::iterator E) {
207 ConstantMapType::iterator MaxCostItr = S;
208 unsigned NumUses = 0;
209 // Use the constant that has the maximum cost as base constant.
210 for (ConstantMapType::iterator I = S; I != E; ++I) {
211 NumUses += I->second.Uses.size();
212 if (I->second.CumulativeCost > MaxCostItr->second.CumulativeCost)
213 MaxCostItr = I;
214 }
215
216 // Don't hoist constants that have only one use.
217 if (NumUses <= 1)
218 return;
219
220 ConstantInfo CI;
221 CI.BaseConstant = MaxCostItr->first;
222 Type *Ty = CI.BaseConstant->getType();
223 // Rebase the constants with respect to the base constant.
224 for (ConstantMapType::iterator I = S; I != E; ++I) {
225 APInt Diff = I->first->getValue() - CI.BaseConstant->getValue();
226 ConstantInfo::RebasedConstantInfo RCI;
227 RCI.OriginalConstant = I->first;
228 RCI.Offset = ConstantInt::get(Ty, Diff);
229 RCI.Uses = llvm_move(I->second.Uses);
230 CI.RebasedConstants.push_back(RCI);
231 }
232 Constants.push_back(CI);
233 }
234
235 /// \brief Finds and combines constants that can be easily rematerialized with
236 /// an add from a common base constant.
237 void ConstantHoisting::FindBaseConstants() {
238 // Sort the constants by value and type. This invalidates the mapping.
239 std::sort(ConstantMap.begin(), ConstantMap.end(), ConstantMapLessThan);
240
241 // Simple linear scan through the sorted constant map for viable merge
242 // candidates.
243 ConstantMapType::iterator MinValItr = ConstantMap.begin();
244 for (ConstantMapType::iterator I = llvm::next(ConstantMap.begin()),
245 E = ConstantMap.end(); I != E; ++I) {
246 if (MinValItr->first->getType() == I->first->getType()) {
247 // Check if the constant is in range of an add with immediate.
248 APInt Diff = I->first->getValue() - MinValItr->first->getValue();
249 if ((Diff.getBitWidth() <= 64) &&
250 TTI->isLegalAddImmediate(Diff.getSExtValue()))
251 continue;
252 }
253 // We either have now a different constant type or the constant is not in
254 // range of an add with immediate anymore.
255 FindAndMakeBaseConstant(MinValItr, I);
256 // Start a new base constant search.
257 MinValItr = I;
258 }
259 // Finalize the last base constant search.
260 FindAndMakeBaseConstant(MinValItr, ConstantMap.end());
261 }
262
263 /// \brief Records the basic block of the instruction or all basic blocks of the
264 /// users of the constant expression.
265 static void CollectBasicBlocks(SmallPtrSet &BBs, User *U) {
266 if (Instruction *I = dyn_cast(U))
267 BBs.insert(I->getParent());
268 else if (ConstantExpr *CE = dyn_cast(U))
269 // Find all users of this constant expression.
270 for (Value::use_iterator UU = CE->use_begin(), E = CE->use_end();
271 UU != E; ++UU)
272 // Only record users that are instructions. We don't want to go down a
273 // nested constant expression chain.
274 if (Instruction *I = dyn_cast(*UU))
275 BBs.insert(I->getParent());
276 }
277
278 /// \brief Find an insertion point that dominates all uses.
279 Instruction *ConstantHoisting::
280 FindConstantInsertionPoint(Function &F, const ConstantInfo &CI) const {
281 BasicBlock *Entry = &F.getEntryBlock();
282
283 // Collect all basic blocks.
284 SmallPtrSet BBs;
285 ConstantInfo::RebasedConstantListType::const_iterator RCI, RCE;
286 for (RCI = CI.RebasedConstants.begin(), RCE = CI.RebasedConstants.end();
287 RCI != RCE; ++RCI)
288 for (SmallVectorImpl::const_iterator U = RCI->Uses.begin(),
289 E = RCI->Uses.end(); U != E; ++U)
290 CollectBasicBlocks(BBs, *U);
291
292 if (BBs.count(Entry))
293 return Entry->getFirstInsertionPt();
294
295 while (BBs.size() >= 2) {
296 BasicBlock *BB, *BB1, *BB2;
297 BB1 = *BBs.begin();
298 BB2 = *llvm::next(BBs.begin());
299 BB = DT->findNearestCommonDominator(BB1, BB2);
300 if (BB == Entry)
301 return Entry->getFirstInsertionPt();
302 BBs.erase(BB1);
303 BBs.erase(BB2);
304 BBs.insert(BB);
305 }
306 assert((BBs.size() == 1) && "Expected only one element.");
307 return (*BBs.begin())->getFirstInsertionPt();
308 }
309
310 /// \brief Emit materialization code for all rebased constants and update their
311 /// users.
312 void ConstantHoisting::EmitBaseConstants(Function &F, User *U,
313 Instruction *Base, Constant *Offset,
314 ConstantInt *OriginalConstant) {
315 if (Instruction *I = dyn_cast(U)) {
316 Instruction *Mat = Base;
317 if (!Offset->isNullValue()) {
318 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
319 "const_mat", I);
320
321 // Use the same debug location as the instruction we are about to update.
322 Mat->setDebugLoc(I->getDebugLoc());
323
324 DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
325 << " + " << *Offset << ") in BB "
326 << I->getParent()->getName() << '\n' << *Mat << '\n');
327 }
328 DEBUG(dbgs() << "Update: " << *I << '\n');
329 I->replaceUsesOfWith(OriginalConstant, Mat);
330 DEBUG(dbgs() << "To: " << *I << '\n');
331 return;
332 }
333 assert(isa(U) && "Expected a ConstantExpr.");
334 ConstantExpr *CE = cast(U);
335 for (Value::use_iterator UU = CE->use_begin(), E = CE->use_end();
336 UU != E; ++UU) {
337 // We only handel instructions here and won't walk down a ConstantExpr chain
338 // to replace all ConstExpr with instructions.
339 if (Instruction *I = dyn_cast(*UU)) {
340 Instruction *Mat = Base;
341 if (!Offset->isNullValue()) {
342 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
343 "const_mat", I);
344
345 // Use the same debug location as the instruction we are about to
346 // update.
347 Mat->setDebugLoc(I->getDebugLoc());
348
349 DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
350 << " + " << *Offset << ") in BB "
351 << I->getParent()->getName() << '\n' << *Mat << '\n');
352 }
353 Instruction *ICE = CE->getAsInstruction();
354 ICE->replaceUsesOfWith(OriginalConstant, Mat);
355 ICE->insertBefore(I);
356
357 // Use the same debug location as the instruction we are about to update.
358 ICE->setDebugLoc(I->getDebugLoc());
359
360 DEBUG(dbgs() << "Create instruction: " << *ICE << '\n');
361 DEBUG(dbgs() << "Update: " << *I << '\n');
362 I->replaceUsesOfWith(CE, ICE);
363 DEBUG(dbgs() << "To: " << *I << '\n');
364 }
365 }
366 }
367
368 /// \brief Hoist and hide the base constant behind a bitcast and emit
369 /// materialization code for derived constants.
370 bool ConstantHoisting::EmitBaseConstants(Function &F) {
371 bool MadeChange = false;
372 SmallVectorImpl::iterator CI, CE;
373 for (CI = Constants.begin(), CE = Constants.end(); CI != CE; ++CI) {
374 // Hoist and hide the base constant behind a bitcast.
375 Instruction *IP = FindConstantInsertionPoint(F, *CI);
376 IntegerType *Ty = CI->BaseConstant->getType();
377 Instruction *Base = new BitCastInst(CI->BaseConstant, Ty, "const", IP);
378 DEBUG(dbgs() << "Hoist constant (" << *CI->BaseConstant << ") to BB "
379 << IP->getParent()->getName() << '\n');
380 NumConstantsHoisted++;
381
382 // Emit materialization code for all rebased constants.
383 ConstantInfo::RebasedConstantListType::iterator RCI, RCE;
384 for (RCI = CI->RebasedConstants.begin(), RCE = CI->RebasedConstants.end();
385 RCI != RCE; ++RCI) {
386 NumConstantsRebased++;
387 for (SmallVectorImpl::iterator U = RCI->Uses.begin(),
388 E = RCI->Uses.end(); U != E; ++U)
389 EmitBaseConstants(F, *U, Base, RCI->Offset, RCI->OriginalConstant);
390 }
391
392 // Use the same debug location as the last user of the constant.
393 assert(!Base->use_empty() && "The use list is empty!?");
394 assert(isa(Base->use_back()) &&
395 "All uses should be instructions.");
396 Base->setDebugLoc(cast(Base->use_back())->getDebugLoc());
397
398 // Correct for base constant, which we counted above too.
399 NumConstantsRebased--;
400 MadeChange = true;
401 }
402 return MadeChange;
403 }
404
405 /// \brief Optimize expensive integer constants in the given function.
406 bool ConstantHoisting::OptimizeConstants(Function &F) {
407 bool MadeChange = false;
408
409 // Collect all constant candidates.
410 CollectConstants(F);
411
412 // There are no constants to worry about.
413 if (ConstantMap.empty())
414 return MadeChange;
415
416 // Combine constants that can be easily materialized with an add from a common
417 // base constant.
418 FindBaseConstants();
419
420 // Finaly hoist the base constant and emit materializating code for dependent
421 // constants.
422 MadeChange |= EmitBaseConstants(F);
423
424 ConstantMap.clear();
425 Constants.clear();
426
427 return MadeChange;
428 }
2929 initializeADCEPass(Registry);
3030 initializeSampleProfileLoaderPass(Registry);
3131 initializeCodeGenPreparePass(Registry);
32 initializeConstantHoistingPass(Registry);
3332 initializeConstantPropagationPass(Registry);
3433 initializeCorrelatedValuePropagationPass(Registry);
3534 initializeDCEPass(Registry);
3737 define void @t2(i8* nocapture %C) nounwind {
3838 entry:
3939 ; CHECK-LABEL: t2:
40 ; CHECK: movw [[REG2:r[0-9]+]], #16716
41 ; CHECK: movt [[REG2:r[0-9]+]], #72
40 ; CHECK: ldr [[REG2:r[0-9]+]], [r1, #32]
4241 ; CHECK: str [[REG2]], [r0, #32]
4342 ; CHECK: vld1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r1]
4443 ; CHECK: vst1.8 {d{{[0-9]+}}, d{{[0-9]+}}}, [r0]
7978 ; CHECK: strb [[REG5]], [r0, #6]
8079 ; CHECK: movw [[REG6:r[0-9]+]], #21587
8180 ; CHECK: strh [[REG6]], [r0, #4]
82 ; CHECK: movw [[REG7:r[0-9]+]], #18500
83 ; CHECK: movt [[REG7:r[0-9]+]], #22866
81 ; CHECK: ldr [[REG7:r[0-9]+]],
8482 ; CHECK: str [[REG7]]
8583 tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([7 x i8]* @.str5, i64 0, i64 0), i64 7, i32 1, i1 false)
8684 ret void
+0
-53
test/CodeGen/X86/large-constants.ll less more
None ; RUN: llc < %s -mtriple=x86_64-darwin -mcpu=corei7 | grep movabsq | count 2
1
2 define i64 @constant_hoisting(i64 %o0, i64 %o1, i64 %o2, i64 %o3, i64 %o4, i64 %o5) {
3 entry:
4 %l0 = and i64 %o0, -281474976710654
5 %c0 = icmp ne i64 %l0, 0
6 br i1 %c0, label %fail, label %bb1
7
8 bb1:
9 %l1 = and i64 %o1, -281474976710654
10 %c1 = icmp ne i64 %l1, 0
11 br i1 %c1, label %fail, label %bb2
12
13 bb2:
14 %l2 = and i64 %o2, -281474976710654
15 %c2 = icmp ne i64 %l2, 0
16 br i1 %c2, label %fail, label %bb3
17
18 bb3:
19 %l3 = and i64 %o3, -281474976710654
20 %c3 = icmp ne i64 %l3, 0
21 br i1 %c3, label %fail, label %bb4
22
23 bb4:
24 %l4 = and i64 %o4, -281474976710653
25 %c4 = icmp ne i64 %l4, 0
26 br i1 %c4, label %fail, label %bb5
27
28 bb5:
29 %l5 = and i64 %o5, -281474976710652
30 %c5 = icmp ne i64 %l5, 0
31 br i1 %c5, label %fail, label %bb6
32
33 bb6:
34 ret i64 %l5
35
36 fail:
37 ret i64 -1
38 }
39
40 define void @constant_expressions() {
41 entry:
42 %0 = load i64* inttoptr (i64 add (i64 51250129900, i64 0) to i64*)
43 %1 = load i64* inttoptr (i64 add (i64 51250129900, i64 8) to i64*)
44 %2 = load i64* inttoptr (i64 add (i64 51250129900, i64 16) to i64*)
45 %3 = load i64* inttoptr (i64 add (i64 51250129900, i64 24) to i64*)
46 %4 = add i64 %0, %1
47 %5 = add i64 %2, %3
48 %6 = add i64 %4, %5
49 store i64 %6, i64* inttoptr (i64 add (i64 51250129900, i64 0) to i64*)
50 ret void
51 }
52