llvm.org GIT mirror llvm / 2d1be87
Expand GEPs in ScalarEvolution expressions. SCEV expressions can now have pointer types, though in contrast to C pointer types, SCEV addition is never implicitly scaled. This not only eliminates the need for special code like IndVars' EliminatePointerRecurrence and LSR's own GEP expansion code, it also does a better job because it lets the normal optimizations handle pointer expressions just like integer expressions. Also, since LLVM IR GEPs can't directly index into multi-dimensional VLAs, moving the GEP analysis out of client code and into the SCEV framework makes it easier for clients to handle multi-dimensional VLAs the same way as other arrays. Some existing regression tests show improved optimization. test/CodeGen/ARM/2007-03-13-InstrSched.ll in particular improved to the point where if-conversion started kicking in; I turned it off for this test to preserve the intent of the test. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@69258 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 11 years ago
10 changed file(s) with 455 addition(s) and 458 deletion(s). Raw diff Collapse all Expand all
3131 class Type;
3232 class SCEVHandle;
3333 class ScalarEvolution;
34 class TargetData;
3435
3536 /// SCEV - This class represent an analyzed expression in the program. These
3637 /// are reference counted opaque objects that the client is not allowed to
6970 /// getType - Return the LLVM type of this SCEV expression.
7071 ///
7172 virtual const Type *getType() const = 0;
72
73 /// getBitWidth - Get the bit width of the type, if it has one, 0 otherwise.
74 ///
75 uint32_t getBitWidth() const;
7673
7774 /// isZero - Return true if the expression is a constant zero.
7875 ///
197194 public:
198195 static char ID; // Pass identification, replacement for typeid
199196 ScalarEvolution() : FunctionPass(&ID), Impl(0) {}
197
198 // getTargetData - Return the TargetData object contained in this
199 // ScalarEvolution.
200 const TargetData &getTargetData() const;
200201
201202 /// getSCEV - Return a SCEV expression handle for the full generality of the
202203 /// specified expression.
1919 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
2020
2121 namespace llvm {
22 class TargetData;
23
2224 /// SCEVExpander - This class uses information about analyze scalars to
2325 /// rewrite expressions in canonical form.
2426 ///
2830 struct SCEVExpander : public SCEVVisitor {
2931 ScalarEvolution &SE;
3032 LoopInfo &LI;
33 const TargetData &TD;
3134 std::map InsertedExpressions;
3235 std::set InsertedInstructions;
3336
3538
3639 friend struct SCEVVisitor;
3740 public:
38 SCEVExpander(ScalarEvolution &se, LoopInfo &li) : SE(se), LI(li) {}
41 SCEVExpander(ScalarEvolution &se, LoopInfo &li, const TargetData &td)
42 : SE(se), LI(li), TD(td) {}
3943
4044 LoopInfo &getLoopInfo() const { return LI; }
4145
7478 /// expandCodeFor - Insert code to directly compute the specified SCEV
7579 /// expression into the program. The inserted code is inserted into the
7680 /// specified block.
77 Value *expandCodeFor(SCEVHandle SH, Instruction *IP);
81 Value *expandCodeFor(SCEVHandle SH, const Type *Ty, Instruction *IP);
7882
7983 /// InsertCastOfTo - Insert a cast of V to the specified type, doing what
8084 /// we can to share the casts.
8690 Value *RHS, Instruction *InsertPt);
8791 protected:
8892 Value *expand(SCEV *S);
89
93
9094 Value *visitConstant(SCEVConstant *S) {
9195 return S->getValue();
9296 }
6868 #include "llvm/Analysis/Dominators.h"
6969 #include "llvm/Analysis/LoopInfo.h"
7070 #include "llvm/Assembly/Writer.h"
71 #include "llvm/Target/TargetData.h"
7172 #include "llvm/Transforms/Scalar.h"
7273 #include "llvm/Support/CFG.h"
7374 #include "llvm/Support/CommandLine.h"
7475 #include "llvm/Support/Compiler.h"
7576 #include "llvm/Support/ConstantRange.h"
77 #include "llvm/Support/GetElementPtrTypeIterator.h"
7678 #include "llvm/Support/InstIterator.h"
7779 #include "llvm/Support/ManagedStatic.h"
7880 #include "llvm/Support/MathExtras.h"
7981 #include "llvm/Support/Streams.h"
8082 #include "llvm/ADT/Statistic.h"
83 #include "llvm/ADT/STLExtras.h"
8184 #include
8285 #include
8386 #include
115118 cerr << '\n';
116119 }
117120
118 uint32_t SCEV::getBitWidth() const {
119 if (const IntegerType* ITy = dyn_cast(getType()))
120 return ITy->getBitWidth();
121 return 0;
122 }
123
124121 bool SCEV::isZero() const {
125122 if (const SCEVConstant *SC = dyn_cast(this))
126123 return SC->getValue()->isZero();
195192
196193 SCEVTruncateExpr::SCEVTruncateExpr(const SCEVHandle &op, const Type *ty)
197194 : SCEV(scTruncate), Op(op), Ty(ty) {
198 assert(Op->getType()->isInteger() && Ty->isInteger() &&
195 assert((Op->getType()->isInteger() || isa(Op->getType())) &&
196 (Ty->isInteger() || isa(Ty)) &&
199197 "Cannot truncate non-integer value!");
200 assert(Op->getType()->getPrimitiveSizeInBits() > Ty->getPrimitiveSizeInBits()
201 && "This is not a truncating conversion!");
198 assert((!Op->getType()->isInteger() || !Ty->isInteger() ||
199 Op->getType()->getPrimitiveSizeInBits() >
200 Ty->getPrimitiveSizeInBits()) &&
201 "This is not a truncating conversion!");
202202 }
203203
204204 SCEVTruncateExpr::~SCEVTruncateExpr() {
221221
222222 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const SCEVHandle &op, const Type *ty)
223223 : SCEV(scZeroExtend), Op(op), Ty(ty) {
224 assert(Op->getType()->isInteger() && Ty->isInteger() &&
224 assert((Op->getType()->isInteger() || isa(Op->getType())) &&
225 (Ty->isInteger() || isa(Ty)) &&
225226 "Cannot zero extend non-integer value!");
226227 assert(Op->getType()->getPrimitiveSizeInBits() < Ty->getPrimitiveSizeInBits()
227228 && "This is not an extending conversion!");
247248
248249 SCEVSignExtendExpr::SCEVSignExtendExpr(const SCEVHandle &op, const Type *ty)
249250 : SCEV(scSignExtend), Op(op), Ty(ty) {
250 assert(Op->getType()->isInteger() && Ty->isInteger() &&
251 assert((Op->getType()->isInteger() || isa(Op->getType())) &&
252 (Ty->isInteger() || isa(Ty)) &&
251253 "Cannot sign extend non-integer value!");
252254 assert(Op->getType()->getPrimitiveSizeInBits() < Ty->getPrimitiveSizeInBits()
253255 && "This is not an extending conversion!");
435437 }
436438
437439 void SCEVUnknown::print(std::ostream &OS) const {
440 if (isa(V->getType()))
441 OS << "(ptrtoint " << *V->getType() << " ";
438442 WriteAsOperand(OS, V, false);
443 if (isa(V->getType()))
444 OS << " to iPTR)";
439445 }
440446
441447 //===----------------------------------------------------------------------===//
503509 // Simple SCEV method implementations
504510 //===----------------------------------------------------------------------===//
505511
506 /// getIntegerSCEV - Given an integer or FP type, create a constant for the
507 /// specified signed integer value and return a SCEV for the constant.
508 SCEVHandle ScalarEvolution::getIntegerSCEV(int Val, const Type *Ty) {
509 Constant *C;
510 if (Val == 0)
511 C = Constant::getNullValue(Ty);
512 else if (Ty->isFloatingPoint())
513 C = ConstantFP::get(APFloat(Ty==Type::FloatTy ? APFloat::IEEEsingle :
514 APFloat::IEEEdouble, Val));
515 else
516 C = ConstantInt::get(Ty, Val);
517 return getUnknown(C);
518 }
519
520 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
521 ///
522 SCEVHandle ScalarEvolution::getNegativeSCEV(const SCEVHandle &V) {
523 if (SCEVConstant *VC = dyn_cast(V))
524 return getUnknown(ConstantExpr::getNeg(VC->getValue()));
525
526 return getMulExpr(V, getConstant(ConstantInt::getAllOnesValue(V->getType())));
527 }
528
529 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
530 SCEVHandle ScalarEvolution::getNotSCEV(const SCEVHandle &V) {
531 if (SCEVConstant *VC = dyn_cast(V))
532 return getUnknown(ConstantExpr::getNot(VC->getValue()));
533
534 SCEVHandle AllOnes = getConstant(ConstantInt::getAllOnesValue(V->getType()));
535 return getMinusSCEV(AllOnes, V);
536 }
537
538 /// getMinusSCEV - Return a SCEV corresponding to LHS - RHS.
539 ///
540 SCEVHandle ScalarEvolution::getMinusSCEV(const SCEVHandle &LHS,
541 const SCEVHandle &RHS) {
542 // X - Y --> X + -Y
543 return getAddExpr(LHS, getNegativeSCEV(RHS));
544 }
545
546
547512 /// BinomialCoefficient - Compute BC(It, K). The result has width W.
548513 // Assume, K > 0.
549514 static SCEVHandle BinomialCoefficient(SCEVHandle It, unsigned K,
550515 ScalarEvolution &SE,
551 const IntegerType* ResultTy) {
516 const Type* ResultTy) {
552517 // Handle the simplest case efficiently.
553518 if (K == 1)
554519 return SE.getTruncateOrZeroExtend(It, ResultTy);
607572 if (K > 1000)
608573 return new SCEVCouldNotCompute();
609574
610 unsigned W = ResultTy->getBitWidth();
575 unsigned W = SE.getTargetData().getTypeSizeInBits(ResultTy);
611576
612577 // Calculate K! / 2^T and T; we divide out the factors of two before
613578 // multiplying for calculating K! / 2^T to avoid overflow.
671636 // The computation is correct in the face of overflow provided that the
672637 // multiplication is performed _after_ the evaluation of the binomial
673638 // coefficient.
674 SCEVHandle Coeff = BinomialCoefficient(It, i, SE,
675 cast(getType()));
639 SCEVHandle Coeff = BinomialCoefficient(It, i, SE, getType());
676640 if (isa(Coeff))
677641 return Coeff;
678642
710674 }
711675
712676 SCEVHandle ScalarEvolution::getZeroExtendExpr(const SCEVHandle &Op, const Type *Ty) {
713 if (SCEVConstant *SC = dyn_cast(Op))
714 return getUnknown(
715 ConstantExpr::getZExt(SC->getValue(), Ty));
677 if (SCEVConstant *SC = dyn_cast(Op)) {
678 const Type *IntTy = Ty;
679 if (isa(IntTy)) IntTy = getTargetData().getIntPtrType();
680 Constant *C = ConstantExpr::getZExt(SC->getValue(), IntTy);
681 if (IntTy != Ty) C = ConstantExpr::getIntToPtr(C, Ty);
682 return getUnknown(C);
683 }
716684
717685 // FIXME: If the input value is a chrec scev, and we can prove that the value
718686 // did not overflow the old, smaller, value, we can zero extend all of the
725693 }
726694
727695 SCEVHandle ScalarEvolution::getSignExtendExpr(const SCEVHandle &Op, const Type *Ty) {
728 if (SCEVConstant *SC = dyn_cast(Op))
729 return getUnknown(
730 ConstantExpr::getSExt(SC->getValue(), Ty));
696 if (SCEVConstant *SC = dyn_cast(Op)) {
697 const Type *IntTy = Ty;
698 if (isa(IntTy)) IntTy = getTargetData().getIntPtrType();
699 Constant *C = ConstantExpr::getSExt(SC->getValue(), IntTy);
700 if (IntTy != Ty) C = ConstantExpr::getIntToPtr(C, Ty);
701 return getUnknown(C);
702 }
731703
732704 // FIXME: If the input value is a chrec scev, and we can prove that the value
733705 // did not overflow the old, smaller, value, we can sign extend all of the
737709 SCEVSignExtendExpr *&Result = (*SCEVSignExtends)[std::make_pair(Op, Ty)];
738710 if (Result == 0) Result = new SCEVSignExtendExpr(Op, Ty);
739711 return Result;
740 }
741
742 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion
743 /// of the input value to the specified type. If the type must be
744 /// extended, it is zero extended.
745 SCEVHandle ScalarEvolution::getTruncateOrZeroExtend(const SCEVHandle &V,
746 const Type *Ty) {
747 const Type *SrcTy = V->getType();
748 assert(SrcTy->isInteger() && Ty->isInteger() &&
749 "Cannot truncate or zero extend with non-integer arguments!");
750 if (SrcTy->getPrimitiveSizeInBits() == Ty->getPrimitiveSizeInBits())
751 return V; // No conversion
752 if (SrcTy->getPrimitiveSizeInBits() > Ty->getPrimitiveSizeInBits())
753 return getTruncateExpr(V, Ty);
754 return getZeroExtendExpr(V, Ty);
755 }
756
757 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion
758 /// of the input value to the specified type. If the type must be
759 /// extended, it is sign extended.
760 SCEVHandle ScalarEvolution::getTruncateOrSignExtend(const SCEVHandle &V,
761 const Type *Ty) {
762 const Type *SrcTy = V->getType();
763 assert(SrcTy->isInteger() && Ty->isInteger() &&
764 "Cannot truncate or sign extend with non-integer arguments!");
765 if (SrcTy->getPrimitiveSizeInBits() == Ty->getPrimitiveSizeInBits())
766 return V; // No conversion
767 if (SrcTy->getPrimitiveSizeInBits() > Ty->getPrimitiveSizeInBits())
768 return getTruncateExpr(V, Ty);
769 return getSignExtendExpr(V, Ty);
770712 }
771713
772714 // get - Get a canonical add expression, or something simpler if possible.
13841326 SCEVHandle ScalarEvolution::getUnknown(Value *V) {
13851327 if (ConstantInt *CI = dyn_cast(V))
13861328 return getConstant(CI);
1329 if (isa(V))
1330 return getIntegerSCEV(0, V->getType());
13871331 SCEVUnknown *&Result = (*SCEVUnknowns)[V];
13881332 if (Result == 0) Result = new SCEVUnknown(V);
13891333 return Result;
14101354 ///
14111355 LoopInfo &LI;
14121356
1357 /// TD - The target data information for the target we are targetting.
1358 ///
1359 TargetData &TD;
1360
14131361 /// UnknownValue - This SCEV is used to represent unknown trip counts and
14141362 /// things.
14151363 SCEVHandle UnknownValue;
14301378 std::map ConstantEvolutionLoopExitValue;
14311379
14321380 public:
1433 ScalarEvolutionsImpl(ScalarEvolution &se, Function &f, LoopInfo &li)
1434 : SE(se), F(f), LI(li), UnknownValue(new SCEVCouldNotCompute()) {}
1381 ScalarEvolutionsImpl(ScalarEvolution &se, Function &f, LoopInfo &li,
1382 TargetData &td)
1383 : SE(se), F(f), LI(li), TD(td), UnknownValue(new SCEVCouldNotCompute()) {}
1384
1385 /// getIntegerSCEV - Given an integer or FP type, create a constant for the
1386 /// specified signed integer value and return a SCEV for the constant.
1387 SCEVHandle getIntegerSCEV(int Val, const Type *Ty);
1388
1389 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
1390 ///
1391 SCEVHandle getNegativeSCEV(const SCEVHandle &V);
1392
1393 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
1394 ///
1395 SCEVHandle getNotSCEV(const SCEVHandle &V);
1396
1397 /// getMinusSCEV - Return a SCEV corresponding to LHS - RHS.
1398 ///
1399 SCEVHandle getMinusSCEV(const SCEVHandle &LHS, const SCEVHandle &RHS);
1400
1401 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion
1402 /// of the input value to the specified type. If the type must be extended,
1403 /// it is zero extended.
1404 SCEVHandle getTruncateOrZeroExtend(const SCEVHandle &V, const Type *Ty);
1405
1406 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion
1407 /// of the input value to the specified type. If the type must be extended,
1408 /// it is sign extended.
1409 SCEVHandle getTruncateOrSignExtend(const SCEVHandle &V, const Type *Ty);
14351410
14361411 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
14371412 /// expression and create a new one.
14911466 /// that no dangling references are left around.
14921467 void deleteValueFromRecords(Value *V);
14931468
1469 /// getTargetData - Return the TargetData.
1470 const TargetData &getTargetData() const;
1471
14941472 private:
14951473 /// createSCEV - We know that there is no SCEV for the specified value.
14961474 /// Analyze the expression.
15911569 }
15921570 }
15931571
1572 const TargetData &ScalarEvolutionsImpl::getTargetData() const {
1573 return TD;
1574 }
15941575
15951576 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
15961577 /// expression and create a new one.
16021583 SCEVHandle S = createSCEV(V);
16031584 Scalars.insert(std::make_pair(V, S));
16041585 return S;
1586 }
1587
1588 /// getIntegerSCEV - Given an integer or FP type, create a constant for the
1589 /// specified signed integer value and return a SCEV for the constant.
1590 SCEVHandle ScalarEvolutionsImpl::getIntegerSCEV(int Val, const Type *Ty) {
1591 if (isa(Ty))
1592 Ty = TD.getIntPtrType();
1593 Constant *C;
1594 if (Val == 0)
1595 C = Constant::getNullValue(Ty);
1596 else if (Ty->isFloatingPoint())
1597 C = ConstantFP::get(APFloat(Ty==Type::FloatTy ? APFloat::IEEEsingle :
1598 APFloat::IEEEdouble, Val));
1599 else
1600 C = ConstantInt::get(Ty, Val);
1601 return SE.getUnknown(C);
1602 }
1603
1604 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
1605 ///
1606 SCEVHandle ScalarEvolutionsImpl::getNegativeSCEV(const SCEVHandle &V) {
1607 if (SCEVConstant *VC = dyn_cast(V))
1608 return SE.getUnknown(ConstantExpr::getNeg(VC->getValue()));
1609
1610 const Type *Ty = V->getType();
1611 if (isa(Ty))
1612 Ty = TD.getIntPtrType();
1613 return SE.getMulExpr(V, SE.getConstant(ConstantInt::getAllOnesValue(Ty)));
1614 }
1615
1616 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
1617 SCEVHandle ScalarEvolutionsImpl::getNotSCEV(const SCEVHandle &V) {
1618 if (SCEVConstant *VC = dyn_cast(V))
1619 return SE.getUnknown(ConstantExpr::getNot(VC->getValue()));
1620
1621 const Type *Ty = V->getType();
1622 if (isa(Ty))
1623 Ty = TD.getIntPtrType();
1624 SCEVHandle AllOnes = SE.getConstant(ConstantInt::getAllOnesValue(Ty));
1625 return getMinusSCEV(AllOnes, V);
1626 }
1627
1628 /// getMinusSCEV - Return a SCEV corresponding to LHS - RHS.
1629 ///
1630 SCEVHandle ScalarEvolutionsImpl::getMinusSCEV(const SCEVHandle &LHS,
1631 const SCEVHandle &RHS) {
1632 // X - Y --> X + -Y
1633 return SE.getAddExpr(LHS, SE.getNegativeSCEV(RHS));
1634 }
1635
1636 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the
1637 /// input value to the specified type. If the type must be extended, it is zero
1638 /// extended.
1639 SCEVHandle
1640 ScalarEvolutionsImpl::getTruncateOrZeroExtend(const SCEVHandle &V,
1641 const Type *Ty) {
1642 const Type *SrcTy = V->getType();
1643 assert((SrcTy->isInteger() || isa(SrcTy)) &&
1644 (Ty->isInteger() || isa(Ty)) &&
1645 "Cannot truncate or zero extend with non-integer arguments!");
1646 if (TD.getTypeSizeInBits(SrcTy) == TD.getTypeSizeInBits(Ty))
1647 return V; // No conversion
1648 if (TD.getTypeSizeInBits(SrcTy) > TD.getTypeSizeInBits(Ty))
1649 return SE.getTruncateExpr(V, Ty);
1650 return SE.getZeroExtendExpr(V, Ty);
1651 }
1652
1653 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the
1654 /// input value to the specified type. If the type must be extended, it is sign
1655 /// extended.
1656 SCEVHandle
1657 ScalarEvolutionsImpl::getTruncateOrSignExtend(const SCEVHandle &V,
1658 const Type *Ty) {
1659 const Type *SrcTy = V->getType();
1660 assert((SrcTy->isInteger() || isa(SrcTy)) &&
1661 (Ty->isInteger() || isa(Ty)) &&
1662 "Cannot truncate or zero extend with non-integer arguments!");
1663 if (TD.getTypeSizeInBits(SrcTy) == TD.getTypeSizeInBits(Ty))
1664 return V; // No conversion
1665 if (TD.getTypeSizeInBits(SrcTy) > TD.getTypeSizeInBits(Ty))
1666 return SE.getTruncateExpr(V, Ty);
1667 return SE.getSignExtendExpr(V, Ty);
16051668 }
16061669
16071670 /// ReplaceSymbolicValueWithConcrete - This looks up the computed SCEV value for
17271790 /// guaranteed to end in (at every loop iteration). It is, at the same time,
17281791 /// the minimum number of times S is divisible by 2. For example, given {4,+,8}
17291792 /// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S.
1730 static uint32_t GetMinTrailingZeros(SCEVHandle S) {
1793 static uint32_t GetMinTrailingZeros(SCEVHandle S, const TargetData &TD) {
17311794 if (SCEVConstant *C = dyn_cast(S))
17321795 return C->getValue()->getValue().countTrailingZeros();
17331796
17341797 if (SCEVTruncateExpr *T = dyn_cast(S))
1735 return std::min(GetMinTrailingZeros(T->getOperand()), T->getBitWidth());
1798 return std::min(GetMinTrailingZeros(T->getOperand(), TD),
1799 (uint32_t)TD.getTypeSizeInBits(T->getType()));
17361800
17371801 if (SCEVZeroExtendExpr *E = dyn_cast(S)) {
1738 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
1739 return OpRes == E->getOperand()->getBitWidth() ? E->getBitWidth() : OpRes;
1802 uint32_t OpRes = GetMinTrailingZeros(E->getOperand(), TD);
1803 return OpRes == TD.getTypeSizeInBits(E->getOperand()->getType()) ?
1804 TD.getTypeSizeInBits(E->getOperand()->getType()) : OpRes;
17401805 }
17411806
17421807 if (SCEVSignExtendExpr *E = dyn_cast(S)) {
1743 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
1744 return OpRes == E->getOperand()->getBitWidth() ? E->getBitWidth() : OpRes;
1808 uint32_t OpRes = GetMinTrailingZeros(E->getOperand(), TD);
1809 return OpRes == TD.getTypeSizeInBits(E->getOperand()->getType()) ?
1810 TD.getTypeSizeInBits(E->getOperand()->getType()) : OpRes;
17451811 }
17461812
17471813 if (SCEVAddExpr *A = dyn_cast(S)) {
17481814 // The result is the min of all operands results.
1749 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
1815 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0), TD);
17501816 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
1751 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
1817 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i), TD));
17521818 return MinOpRes;
17531819 }
17541820
17551821 if (SCEVMulExpr *M = dyn_cast(S)) {
17561822 // The result is the sum of all operands results.
1757 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
1758 uint32_t BitWidth = M->getBitWidth();
1823 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0), TD);
1824 uint32_t BitWidth = TD.getTypeSizeInBits(M->getType());
17591825 for (unsigned i = 1, e = M->getNumOperands();
17601826 SumOpRes != BitWidth && i != e; ++i)
1761 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)),
1827 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i), TD),
17621828 BitWidth);
17631829 return SumOpRes;
17641830 }
17651831
17661832 if (SCEVAddRecExpr *A = dyn_cast(S)) {
17671833 // The result is the min of all operands results.
1768 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
1834 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0), TD);
17691835 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
1770 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
1836 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i), TD));
17711837 return MinOpRes;
17721838 }
17731839
17741840 if (SCEVSMaxExpr *M = dyn_cast(S)) {
17751841 // The result is the min of all operands results.
1776 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
1842 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0), TD);
17771843 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
1778 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
1844 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i), TD));
17791845 return MinOpRes;
17801846 }
17811847
17821848 if (SCEVUMaxExpr *M = dyn_cast(S)) {
17831849 // The result is the min of all operands results.
1784 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
1850 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0), TD);
17851851 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
1786 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
1852 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i), TD));
17871853 return MinOpRes;
17881854 }
17891855
17951861 /// Analyze the expression.
17961862 ///
17971863 SCEVHandle ScalarEvolutionsImpl::createSCEV(Value *V) {
1798 if (!isa(V->getType()))
1864 if (!isa(V->getType()) &&
1865 !isa(V->getType()))
17991866 return SE.getUnknown(V);
1800
1867
18011868 unsigned Opcode = Instruction::UserOp1;
18021869 if (Instruction *I = dyn_cast(V))
18031870 Opcode = I->getOpcode();
18301897 if (ConstantInt *CI = dyn_cast(U->getOperand(1))) {
18311898 SCEVHandle LHS = getSCEV(U->getOperand(0));
18321899 const APInt &CIVal = CI->getValue();
1833 if (GetMinTrailingZeros(LHS) >=
1900 if (GetMinTrailingZeros(LHS, TD) >=
18341901 (CIVal.getBitWidth() - CIVal.countLeadingZeros()))
18351902 return SE.getAddExpr(LHS, getSCEV(U->getOperand(1)));
18361903 }
18801947
18811948 case Instruction::BitCast:
18821949 // BitCasts are no-op casts so we just eliminate the cast.
1883 if (U->getType()->isInteger() &&
1884 U->getOperand(0)->getType()->isInteger())
1950 if ((U->getType()->isInteger() ||
1951 isa(U->getType())) &&
1952 (U->getOperand(0)->getType()->isInteger() ||
1953 isa(U->getOperand(0)->getType())))
18851954 return getSCEV(U->getOperand(0));
18861955 break;
1956
1957 case Instruction::IntToPtr:
1958 return getTruncateOrZeroExtend(getSCEV(U->getOperand(0)),
1959 TD.getIntPtrType());
1960
1961 case Instruction::PtrToInt:
1962 return getTruncateOrZeroExtend(getSCEV(U->getOperand(0)),
1963 U->getType());
1964
1965 case Instruction::GetElementPtr: {
1966 const Type *IntPtrTy = TD.getIntPtrType();
1967 Value *Base = U->getOperand(0);
1968 SCEVHandle TotalOffset = SE.getIntegerSCEV(0, IntPtrTy);
1969 gep_type_iterator GTI = gep_type_begin(U);
1970 for (GetElementPtrInst::op_iterator I = next(U->op_begin()),
1971 E = U->op_end();
1972 I != E; ++I) {
1973 Value *Index = *I;
1974 // Compute the (potentially symbolic) offset in bytes for this index.
1975 if (const StructType *STy = dyn_cast(*GTI++)) {
1976 // For a struct, add the member offset.
1977 const StructLayout &SL = *TD.getStructLayout(STy);
1978 unsigned FieldNo = cast(Index)->getZExtValue();
1979 uint64_t Offset = SL.getElementOffset(FieldNo);
1980 TotalOffset = SE.getAddExpr(TotalOffset,
1981 SE.getIntegerSCEV(Offset, IntPtrTy));
1982 } else {
1983 // For an array, add the element offset, explicitly scaled.
1984 SCEVHandle LocalOffset = getSCEV(Index);
1985 if (!isa(LocalOffset->getType()))
1986 // Getelementptr indicies are signed.
1987 LocalOffset = getTruncateOrSignExtend(LocalOffset,
1988 IntPtrTy);
1989 LocalOffset =
1990 SE.getMulExpr(LocalOffset,
1991 SE.getIntegerSCEV(TD.getTypePaddedSize(*GTI),
1992 IntPtrTy));
1993 TotalOffset = SE.getAddExpr(TotalOffset, LocalOffset);
1994 }
1995 }
1996 return SE.getAddExpr(getSCEV(Base), TotalOffset);
1997 }
18871998
18881999 case Instruction::PHI:
18892000 return createNodeForPHI(cast(U));
23232434 static Constant *EvaluateExpression(Value *V, Constant *PHIVal) {
23242435 if (isa(V)) return PHIVal;
23252436 if (Constant *C = dyn_cast(V)) return C;
2437 if (GlobalValue *GV = dyn_cast(V)) return GV;
23262438 Instruction *I = cast(V);
23272439
23282440 std::vector Operands;
24892601 Operands.push_back(C);
24902602 } else {
24912603 // If any of the operands is non-constant and if they are
2492 // non-integer, don't even try to analyze them with scev techniques.
2493 if (!isa(Op->getType()))
2604 // non-integer and non-pointer, don't even try to analyze them
2605 // with scev techniques.
2606 if (!isa(Op->getType()) &&
2607 !isa(Op->getType()))
24942608 return V;
2495
2609
24962610 SCEVHandle OpV = getSCEVAtScope(getSCEV(Op), L);
24972611 if (SCEVConstant *SC = dyn_cast(OpV))
24982612 Operands.push_back(ConstantExpr::getIntegerCast(SC->getValue(),
30023116
30033117 // First check to see if the range contains zero. If not, the first
30043118 // iteration exits.
3005 if (!Range.contains(APInt(getBitWidth(),0)))
3119 unsigned BitWidth = SE.getTargetData().getTypeSizeInBits(getType());
3120 if (!Range.contains(APInt(BitWidth, 0)))
30063121 return SE.getConstant(ConstantInt::get(getType(),0));
30073122
30083123 if (isAffine()) {
30133128 // the upper value of the range must be the first possible exit value.
30143129 // If A is negative then the lower of the range is the last possible loop
30153130 // value. Also note that we already checked for a full range.
3016 APInt One(getBitWidth(),1);
3131 APInt One(BitWidth,1);
30173132 APInt A = cast(getOperand(1))->getValue()->getValue();
30183133 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower();
30193134
30933208 //===----------------------------------------------------------------------===//
30943209
30953210 bool ScalarEvolution::runOnFunction(Function &F) {
3096 Impl = new ScalarEvolutionsImpl(*this, F, getAnalysis());
3211 Impl = new ScalarEvolutionsImpl(*this, F,
3212 getAnalysis(),
3213 getAnalysis());
30973214 return false;
30983215 }
30993216
31053222 void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const {
31063223 AU.setPreservesAll();
31073224 AU.addRequiredTransitive();
3225 AU.addRequiredTransitive();
3226 }
3227
3228 const TargetData &ScalarEvolution::getTargetData() const {
3229 return ((ScalarEvolutionsImpl*)Impl)->getTargetData();
3230 }
3231
3232 SCEVHandle ScalarEvolution::getIntegerSCEV(int Val, const Type *Ty) {
3233 return ((ScalarEvolutionsImpl*)Impl)->getIntegerSCEV(Val, Ty);
31083234 }
31093235
31103236 SCEVHandle ScalarEvolution::getSCEV(Value *V) const {
31223248 /// the specified value.
31233249 void ScalarEvolution::setSCEV(Value *V, const SCEVHandle &H) {
31243250 ((ScalarEvolutionsImpl*)Impl)->setSCEV(V, H);
3251 }
3252
3253 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
3254 ///
3255 SCEVHandle ScalarEvolution::getNegativeSCEV(const SCEVHandle &V) {
3256 return ((ScalarEvolutionsImpl*)Impl)->getNegativeSCEV(V);
3257 }
3258
3259 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
3260 ///
3261 SCEVHandle ScalarEvolution::getNotSCEV(const SCEVHandle &V) {
3262 return ((ScalarEvolutionsImpl*)Impl)->getNotSCEV(V);
3263 }
3264
3265 /// getMinusSCEV - Return a SCEV corresponding to LHS - RHS.
3266 ///
3267 SCEVHandle ScalarEvolution::getMinusSCEV(const SCEVHandle &LHS,
3268 const SCEVHandle &RHS) {
3269 return ((ScalarEvolutionsImpl*)Impl)->getMinusSCEV(LHS, RHS);
3270 }
3271
3272 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion
3273 /// of the input value to the specified type. If the type must be
3274 /// extended, it is zero extended.
3275 SCEVHandle ScalarEvolution::getTruncateOrZeroExtend(const SCEVHandle &V,
3276 const Type *Ty) {
3277 return ((ScalarEvolutionsImpl*)Impl)->getTruncateOrZeroExtend(V, Ty);
3278 }
3279
3280 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion
3281 /// of the input value to the specified type. If the type must be
3282 /// extended, it is sign extended.
3283 SCEVHandle ScalarEvolution::getTruncateOrSignExtend(const SCEVHandle &V,
3284 const Type *Ty) {
3285 return ((ScalarEvolutionsImpl*)Impl)->getTruncateOrSignExtend(V, Ty);
31253286 }
31263287
31273288
1414
1515 #include "llvm/Analysis/ScalarEvolutionExpander.h"
1616 #include "llvm/Analysis/LoopInfo.h"
17 #include "llvm/Target/TargetData.h"
1718 using namespace llvm;
1819
1920 /// InsertCastOfTo - Insert a cast of V to the specified type, doing what
2021 /// we can to share the casts.
2122 Value *SCEVExpander::InsertCastOfTo(Instruction::CastOps opcode, Value *V,
2223 const Type *Ty) {
24 // Short-circuit unnecessary bitcasts.
25 if (opcode == Instruction::BitCast && V->getType() == Ty)
26 return V;
27
2328 // FIXME: keep track of the cast instruction.
2429 if (Constant *C = dyn_cast(V))
2530 return ConstantExpr::getCast(opcode, C, Ty);
99104 }
100105
101106 Value *SCEVExpander::visitAddExpr(SCEVAddExpr *S) {
107 const Type *Ty = S->getType();
108 if (isa(Ty)) Ty = TD.getIntPtrType();
102109 Value *V = expand(S->getOperand(S->getNumOperands()-1));
110 V = InsertCastOfTo(CastInst::getCastOpcode(V, false, Ty, false), V, Ty);
103111
104112 // Emit a bunch of add instructions
105 for (int i = S->getNumOperands()-2; i >= 0; --i)
106 V = InsertBinop(Instruction::Add, V, expand(S->getOperand(i)),
107 InsertPt);
113 for (int i = S->getNumOperands()-2; i >= 0; --i) {
114 Value *W = expand(S->getOperand(i));
115 W = InsertCastOfTo(CastInst::getCastOpcode(W, false, Ty, false), W, Ty);
116 V = InsertBinop(Instruction::Add, V, W, InsertPt);
117 }
108118 return V;
109119 }
110120
111121 Value *SCEVExpander::visitMulExpr(SCEVMulExpr *S) {
122 const Type *Ty = S->getType();
123 if (isa(Ty)) Ty = TD.getIntPtrType();
112124 int FirstOp = 0; // Set if we should emit a subtract.
113125 if (SCEVConstant *SC = dyn_cast(S->getOperand(0)))
114126 if (SC->getValue()->isAllOnesValue())
116128
117129 int i = S->getNumOperands()-2;
118130 Value *V = expand(S->getOperand(i+1));
131 V = InsertCastOfTo(CastInst::getCastOpcode(V, false, Ty, false), V, Ty);
119132
120133 // Emit a bunch of multiply instructions
121 for (; i >= FirstOp; --i)
122 V = InsertBinop(Instruction::Mul, V, expand(S->getOperand(i)),
123 InsertPt);
134 for (; i >= FirstOp; --i) {
135 Value *W = expand(S->getOperand(i));
136 W = InsertCastOfTo(CastInst::getCastOpcode(W, false, Ty, false), W, Ty);
137 V = InsertBinop(Instruction::Mul, V, W, InsertPt);
138 }
139
124140 // -1 * ... ---> 0 - ...
125141 if (FirstOp == 1)
126 V = InsertBinop(Instruction::Sub, Constant::getNullValue(V->getType()), V,
127 InsertPt);
142 V = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), V, InsertPt);
128143 return V;
129144 }
130145
131146 Value *SCEVExpander::visitUDivExpr(SCEVUDivExpr *S) {
147 const Type *Ty = S->getType();
148 if (isa(Ty)) Ty = TD.getIntPtrType();
149
132150 Value *LHS = expand(S->getLHS());
151 LHS = InsertCastOfTo(CastInst::getCastOpcode(LHS, false, Ty, false), LHS, Ty);
133152 if (SCEVConstant *SC = dyn_cast(S->getRHS())) {
134153 const APInt &RHS = SC->getValue()->getValue();
135154 if (RHS.isPowerOf2())
136155 return InsertBinop(Instruction::LShr, LHS,
137 ConstantInt::get(S->getType(), RHS.logBase2()),
156 ConstantInt::get(Ty, RHS.logBase2()),
138157 InsertPt);
139158 }
140159
141160 Value *RHS = expand(S->getRHS());
161 RHS = InsertCastOfTo(CastInst::getCastOpcode(RHS, false, Ty, false), RHS, Ty);
142162 return InsertBinop(Instruction::UDiv, LHS, RHS, InsertPt);
143163 }
144164
146166 const Type *Ty = S->getType();
147167 const Loop *L = S->getLoop();
148168 // We cannot yet do fp recurrences, e.g. the xform of {X,+,F} --> X+{0,+,F}
149 assert(Ty->isInteger() && "Cannot expand fp recurrences yet!");
169 assert((Ty->isInteger() || isa(Ty)) &&
170 "Cannot expand fp recurrences yet!");
150171
151172 // {X,+,F} --> X + {0,+,F}
152173 if (!S->getStart()->isZero()) {
153174 Value *Start = expand(S->getStart());
175 if (isa(Start->getType()))
176 Start = InsertCastOfTo(Instruction::PtrToInt, Start, TD.getIntPtrType());
154177 std::vector NewOps(S->op_begin(), S->op_end());
155178 NewOps[0] = SE.getIntegerSCEV(0, Ty);
156179 Value *Rest = expand(SE.getAddRecExpr(NewOps, L));
180 if (isa(Rest->getType()))
181 Rest = InsertCastOfTo(Instruction::PtrToInt, Rest, TD.getIntPtrType());
157182
158183 // FIXME: look for an existing add to use.
159184 return InsertBinop(Instruction::Add, Rest, Start, InsertPt);
241266
242267 Value *SCEVExpander::visitTruncateExpr(SCEVTruncateExpr *S) {
243268 Value *V = expand(S->getOperand());
269 if (isa(V->getType()))
270 V = InsertCastOfTo(Instruction::PtrToInt, V, TD.getIntPtrType());
244271 return CastInst::CreateTruncOrBitCast(V, S->getType(), "tmp.", InsertPt);
245272 }
246273
247274 Value *SCEVExpander::visitZeroExtendExpr(SCEVZeroExtendExpr *S) {
248275 Value *V = expand(S->getOperand());
276 if (isa(V->getType()))
277 V = InsertCastOfTo(Instruction::PtrToInt, V, TD.getIntPtrType());
249278 return CastInst::CreateZExtOrBitCast(V, S->getType(), "tmp.", InsertPt);
250279 }
251280
252281 Value *SCEVExpander::visitSignExtendExpr(SCEVSignExtendExpr *S) {
253282 Value *V = expand(S->getOperand());
283 if (isa(V->getType()))
284 V = InsertCastOfTo(Instruction::PtrToInt, V, TD.getIntPtrType());
254285 return CastInst::CreateSExtOrBitCast(V, S->getType(), "tmp.", InsertPt);
255286 }
256287
274305 return LHS;
275306 }
276307
277 Value *SCEVExpander::expandCodeFor(SCEVHandle SH, Instruction *IP) {
308 Value *SCEVExpander::expandCodeFor(SCEVHandle SH, const Type *Ty,
309 Instruction *IP) {
278310 // Expand the code for this SCEV.
311 assert(TD.getTypeSizeInBits(Ty) == TD.getTypeSizeInBits(SH->getType()) &&
312 "non-trivial casts should be done with the SCEVs directly!");
279313 this->InsertPt = IP;
280 return expand(SH);
314 Value *V = expand(SH);
315 return InsertCastOfTo(CastInst::getCastOpcode(V, false, Ty, false), V, Ty);
281316 }
282317
283318 Value *SCEVExpander::expand(SCEV *S) {
4949 #include "llvm/Support/Compiler.h"
5050 #include "llvm/Support/Debug.h"
5151 #include "llvm/Support/GetElementPtrTypeIterator.h"
52 #include "llvm/Target/TargetData.h"
5253 #include "llvm/Transforms/Utils/Local.h"
5354 #include "llvm/Support/CommandLine.h"
5455 #include "llvm/ADT/SmallVector.h"
5859 using namespace llvm;
5960
6061 STATISTIC(NumRemoved , "Number of aux indvars removed");
61 STATISTIC(NumPointer , "Number of pointer indvars promoted");
6262 STATISTIC(NumInserted, "Number of canonical indvars added");
6363 STATISTIC(NumReplaced, "Number of exit values replaced");
6464 STATISTIC(NumLFTR , "Number of loop exit tests replaced");
6666 namespace {
6767 class VISIBILITY_HIDDEN IndVarSimplify : public LoopPass {
6868 LoopInfo *LI;
69 TargetData *TD;
6970 ScalarEvolution *SE;
7071 bool Changed;
7172 public:
8081 AU.addRequiredID(LCSSAID);
8182 AU.addRequiredID(LoopSimplifyID);
8283 AU.addRequired();
84 AU.addRequired();
8385 AU.addPreserved();
8486 AU.addPreservedID(LoopSimplifyID);
8587 AU.addPreservedID(LCSSAID);
9092
9193 void RewriteNonIntegerIVs(Loop *L);
9294
93 void EliminatePointerRecurrence(PHINode *PN, BasicBlock *Preheader,
94 SmallPtrSet &DeadInsts);
9595 void LinearFunctionTestReplace(Loop *L, SCEVHandle BackedgeTakenCount,
9696 Value *IndVar,
9797 BasicBlock *ExitingBlock,
132132 Changed = true;
133133 }
134134 }
135 }
136
137
138 /// EliminatePointerRecurrence - Check to see if this is a trivial GEP pointer
139 /// recurrence. If so, change it into an integer recurrence, permitting
140 /// analysis by the SCEV routines.
141 void IndVarSimplify::EliminatePointerRecurrence(PHINode *PN,
142 BasicBlock *Preheader,
143 SmallPtrSet &DeadInsts) {
144 assert(PN->getNumIncomingValues() == 2 && "Noncanonicalized loop!");
145 unsigned PreheaderIdx = PN->getBasicBlockIndex(Preheader);
146 unsigned BackedgeIdx = PreheaderIdx^1;
147 if (GetElementPtrInst *GEPI =
148 dyn_cast(PN->getIncomingValue(BackedgeIdx)))
149 if (GEPI->getOperand(0) == PN) {
150 assert(GEPI->getNumOperands() == 2 && "GEP types must match!");
151 DOUT << "INDVARS: Eliminating pointer recurrence: " << *GEPI;
152
153 // Okay, we found a pointer recurrence. Transform this pointer
154 // recurrence into an integer recurrence. Compute the value that gets
155 // added to the pointer at every iteration.
156 Value *AddedVal = GEPI->getOperand(1);
157
158 // Insert a new integer PHI node into the top of the block.
159 PHINode *NewPhi = PHINode::Create(AddedVal->getType(),
160 PN->getName()+".rec", PN);
161 NewPhi->addIncoming(Constant::getNullValue(NewPhi->getType()), Preheader);
162
163 // Create the new add instruction.
164 Value *NewAdd = BinaryOperator::CreateAdd(NewPhi, AddedVal,
165 GEPI->getName()+".rec", GEPI);
166 NewPhi->addIncoming(NewAdd, PN->getIncomingBlock(BackedgeIdx));
167
168 // Update the existing GEP to use the recurrence.
169 GEPI->setOperand(0, PN->getIncomingValue(PreheaderIdx));
170
171 // Update the GEP to use the new recurrence we just inserted.
172 GEPI->setOperand(1, NewAdd);
173
174 // If the incoming value is a constant expr GEP, try peeling out the array
175 // 0 index if possible to make things simpler.
176 if (ConstantExpr *CE = dyn_cast(GEPI->getOperand(0)))
177 if (CE->getOpcode() == Instruction::GetElementPtr) {
178 unsigned NumOps = CE->getNumOperands();
179 assert(NumOps > 1 && "CE folding didn't work!");
180 if (CE->getOperand(NumOps-1)->isNullValue()) {
181 // Check to make sure the last index really is an array index.
182 gep_type_iterator GTI = gep_type_begin(CE);
183 for (unsigned i = 1, e = CE->getNumOperands()-1;
184 i != e; ++i, ++GTI)
185 /*empty*/;
186 if (isa(*GTI)) {
187 // Pull the last index out of the constant expr GEP.
188 SmallVector CEIdxs(CE->op_begin()+1, CE->op_end()-1);
189 Constant *NCE = ConstantExpr::getGetElementPtr(CE->getOperand(0),
190 &CEIdxs[0],
191 CEIdxs.size());
192 Value *Idx[2];
193 Idx[0] = Constant::getNullValue(Type::Int32Ty);
194 Idx[1] = NewAdd;
195 GetElementPtrInst *NGEPI = GetElementPtrInst::Create(
196 NCE, Idx, Idx + 2,
197 GEPI->getName(), GEPI);
198 SE->deleteValueFromRecords(GEPI);
199 GEPI->replaceAllUsesWith(NGEPI);
200 GEPI->eraseFromParent();
201 GEPI = NGEPI;
202 }
203 }
204 }
205
206
207 // Finally, if there are any other users of the PHI node, we must
208 // insert a new GEP instruction that uses the pre-incremented version
209 // of the induction amount.
210 if (!PN->use_empty()) {
211 BasicBlock::iterator InsertPos = PN; ++InsertPos;
212 while (isa(InsertPos)) ++InsertPos;
213 Value *PreInc =
214 GetElementPtrInst::Create(PN->getIncomingValue(PreheaderIdx),
215 NewPhi, "", InsertPos);
216 PreInc->takeName(PN);
217 PN->replaceAllUsesWith(PreInc);
218 }
219
220 // Delete the old PHI for sure, and the GEP if its otherwise unused.
221 DeadInsts.insert(PN);
222
223 ++NumPointer;
224 Changed = true;
225 }
226135 }
227136
228137 /// LinearFunctionTestReplace - This method rewrites the exit condition of the
274183
275184 // Expand the code for the iteration count into the preheader of the loop.
276185 BasicBlock *Preheader = L->getLoopPreheader();
277 Value *ExitCnt = Rewriter.expandCodeFor(RHS,
186 Value *ExitCnt = Rewriter.expandCodeFor(RHS, IndVar->getType(),
278187 Preheader->getTerminator());
279188
280189 // Insert a new icmp_ne or icmp_eq instruction before the branch.
306215
307216 // Scan all of the instructions in the loop, looking at those that have
308217 // extra-loop users and which are recurrences.
309 SCEVExpander Rewriter(*SE, *LI);
218 SCEVExpander Rewriter(*SE, *LI, *TD);
310219
311220 // We insert the code into the preheader of the loop if the loop contains
312221 // multiple exit blocks, or in the exit block if there is exactly one.
348257 Value *InVal = PN->getIncomingValue(i);
349258 if (!isa(InVal) ||
350259 // SCEV only supports integer expressions for now.
351 !isa(InVal->getType()))
260 (!isa(InVal->getType()) &&
261 !isa(InVal->getType())))
352262 continue;
353263
354264 // If this pred is for a subloop, not L itself, skip it.
383293 // just reuse it.
384294 Value *&ExitVal = ExitValues[Inst];
385295 if (!ExitVal)
386 ExitVal = Rewriter.expandCodeFor(ExitValue, InsertPt);
296 ExitVal = Rewriter.expandCodeFor(ExitValue, PN->getType(), InsertPt);
387297
388298 DOUT << "INDVARS: RLEV: AfterLoopVal = " << *ExitVal
389299 << " LoopVal = " << *Inst << "\n";
412322 }
413323
414324 void IndVarSimplify::RewriteNonIntegerIVs(Loop *L) {
415 // First step. Check to see if there are any trivial GEP pointer recurrences.
325 // First step. Check to see if there are any floating-point recurrences.
416326 // If there are, change them into integer recurrences, permitting analysis by
417327 // the SCEV routines.
418328 //
419329 BasicBlock *Header = L->getHeader();
420 BasicBlock *Preheader = L->getLoopPreheader();
421330
422331 SmallPtrSet DeadInsts;
423332 for (BasicBlock::iterator I = Header->begin(); isa(I); ++I) {
424333 PHINode *PN = cast(I);
425 if (isa(PN->getType()))
426 EliminatePointerRecurrence(PN, Preheader, DeadInsts);
427 else
428 HandleFloatingPointIV(L, PN, DeadInsts);
429 }
430
431 // If the loop previously had a pointer or floating-point IV, ScalarEvolution
334 HandleFloatingPointIV(L, PN, DeadInsts);
335 }
336
337 // If the loop previously had floating-point IV, ScalarEvolution
432338 // may not have been able to compute a trip count. Now that we've done some
433339 // re-writing, the trip count may be computable.
434340 if (Changed)
441347 /// getEffectiveIndvarType - Determine the widest type that the
442348 /// induction-variable PHINode Phi is cast to.
443349 ///
444 static const Type *getEffectiveIndvarType(const PHINode *Phi) {
350 static const Type *getEffectiveIndvarType(const PHINode *Phi,
351 const TargetData *TD) {
445352 const Type *Ty = Phi->getType();
446353
447354 for (Value::use_const_iterator UI = Phi->use_begin(), UE = Phi->use_end();
452359 else if (const SExtInst *SI = dyn_cast(UI))
453360 CandidateType = SI->getDestTy();
454361 if (CandidateType &&
455 CandidateType->getPrimitiveSizeInBits() >
456 Ty->getPrimitiveSizeInBits())
362 TD->getTypeSizeInBits(CandidateType) > TD->getTypeSizeInBits(Ty))
457363 Ty = CandidateType;
458364 }
459365
481387 static const PHINode *TestOrigIVForWrap(const Loop *L,
482388 const BranchInst *BI,
483389 const Instruction *OrigCond,
390 const TargetData *TD,
484391 bool &NoSignedWrap,
485392 bool &NoUnsignedWrap,
486393 const ConstantInt* &InitialVal,
553460 if (const SExtInst *SI = dyn_cast(CmpLHS)) {
554461 if (!isa(CmpRHS) ||
555462 !cast(CmpRHS)->getValue()
556 .isSignedIntN(IncrInst->getType()->getPrimitiveSizeInBits()))
463 .isSignedIntN(TD->getTypeSizeInBits(IncrInst->getType())))
557464 return 0;
558465 IncrInst = SI->getOperand(0);
559466 }
561468 if (const ZExtInst *ZI = dyn_cast(CmpLHS)) {
562469 if (!isa(CmpRHS) ||
563470 !cast(CmpRHS)->getValue()
564 .isIntN(IncrInst->getType()->getPrimitiveSizeInBits()))
471 .isIntN(TD->getTypeSizeInBits(IncrInst->getType())))
565472 return 0;
566473 IncrInst = ZI->getOperand(0);
567474 }
642549 SE->getAddRecExpr(ExtendedStart, ExtendedStep, L);
643550 if (LargestType != myType)
644551 ExtendedAddRec = SE->getTruncateExpr(ExtendedAddRec, myType);
645 return Rewriter.expandCodeFor(ExtendedAddRec, InsertPt);
552 return Rewriter.expandCodeFor(ExtendedAddRec, myType, InsertPt);
646553 }
647554
648555 static Value *getZeroExtendedTruncVar(const SCEVAddRecExpr *AR,
659566 SE->getAddRecExpr(ExtendedStart, ExtendedStep, L);
660567 if (LargestType != myType)
661568 ExtendedAddRec = SE->getTruncateExpr(ExtendedAddRec, myType);
662 return Rewriter.expandCodeFor(ExtendedAddRec, InsertPt);
569 return Rewriter.expandCodeFor(ExtendedAddRec, myType, InsertPt);
663570 }
664571
665572 /// allUsesAreSameTyped - See whether all Uses of I are instructions
681588
682589 bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
683590 LI = &getAnalysis();
591 TD = &getAnalysis();
684592 SE = &getAnalysis();
685593 Changed = false;
686594
687 // If there are any floating-point or pointer recurrences, attempt to
595 // If there are any floating-point recurrences, attempt to
688596 // transform them to use integer recurrences.
689597 RewriteNonIntegerIVs(L);
690598
711619
712620 for (BasicBlock::iterator I = Header->begin(); isa(I); ++I) {
713621 PHINode *PN = cast(I);
714 if (PN->getType()->isInteger()) { // FIXME: when we have fast-math, enable!
622 if (PN->getType()->isInteger() || isa(PN->getType())) {
715623 SCEVHandle SCEV = SE->getSCEV(PN);
716624 // FIXME: It is an extremely bad idea to indvar substitute anything more
717625 // complex than affine induction variables. Doing so will put expensive
730638 SmallSetVector SizesToInsert;
731639 if (!isa(BackedgeTakenCount)) {
732640 LargestType = BackedgeTakenCount->getType();
733 SizesToInsert.insert(BackedgeTakenCount->getType());
641 if (isa(LargestType))
642 LargestType = TD->getIntPtrType();
643 SizesToInsert.insert(LargestType);
734644 }
735645 for (unsigned i = 0, e = IndVars.size(); i != e; ++i) {
736646 const PHINode *PN = IndVars[i].first;
737 SizesToInsert.insert(PN->getType());
738 const Type *EffTy = getEffectiveIndvarType(PN);
647 const Type *PNTy = PN->getType();
648 if (isa(PNTy)) PNTy = TD->getIntPtrType();
649 SizesToInsert.insert(PNTy);
650 const Type *EffTy = getEffectiveIndvarType(PN, TD);
651 if (isa(EffTy)) EffTy = TD->getIntPtrType();
739652 SizesToInsert.insert(EffTy);
740653 if (!LargestType ||
741 EffTy->getPrimitiveSizeInBits() >
742 LargestType->getPrimitiveSizeInBits())
654 TD->getTypeSizeInBits(EffTy) >
655 TD->getTypeSizeInBits(LargestType))
743656 LargestType = EffTy;
744657 }
745658
746659 // Create a rewriter object which we'll use to transform the code with.
747 SCEVExpander Rewriter(*SE, *LI);
660 SCEVExpander Rewriter(*SE, *LI, *TD);
748661
749662 // Now that we know the largest of of the induction variables in this loop,
750663 // insert a canonical induction variable of the largest size.
768681 if (Instruction *OrigCond = dyn_cast(BI->getCondition())) {
769682 // Determine if the OrigIV will ever undergo overflow.
770683 OrigControllingPHI =
771 TestOrigIVForWrap(L, BI, OrigCond,
684 TestOrigIVForWrap(L, BI, OrigCond, TD,
772685 NoSignedWrap, NoUnsignedWrap,
773686 InitialVal, IncrVal, LimitVal);
774687
803716 while (!IndVars.empty()) {
804717 PHINode *PN = IndVars.back().first;
805718 SCEVAddRecExpr *AR = cast(IndVars.back().second);
806 Value *NewVal = Rewriter.expandCodeFor(AR, InsertPt);
719 Value *NewVal = Rewriter.expandCodeFor(AR, PN->getType(), InsertPt);
807720 DOUT << "INDVARS: Rewrote IV '" << *AR << "' " << *PN
808721 << " into = " << *NewVal << "\n";
809722 NewVal->takeName(PN);
None //===- LoopStrengthReduce.cpp - Strength Reduce GEPs in Loops -------------===//
0 //===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
11 //
22 // The LLVM Compiler Infrastructure
33 //
77 //===----------------------------------------------------------------------===//
88 //
99 // This pass performs a strength reduction on array references inside loops that
10 // have as one or more of their components the loop induction variable. This is
11 // accomplished by creating a new Value to hold the initial value of the array
12 // access for the first iteration, and then creating a new GEP instruction in
13 // the loop to increment the value by the appropriate amount.
10 // have as one or more of their components the loop induction variable.
1411 //
1512 //===----------------------------------------------------------------------===//
1613
132129 /// dependent on random ordering of pointers in the process.
133130 SmallVector StrideOrder;
134131
135 /// GEPlist - A list of the GEP's that have been remembered in the SCEV
136 /// data structures. SCEV does not know to update these when the operands
137 /// of the GEP are changed, which means we cannot leave them live across
138 /// loops.
139 SmallVector GEPlist;
140
141 /// CastedValues - As we need to cast values to uintptr_t, this keeps track
142 /// of the casted version of each value. This is accessed by
143 /// getCastedVersionOf.
144 DenseMap CastedPointers;
145
146132 /// DeadInsts - Keep track of instructions we may have made dead, so that
147133 /// we can remove them after we are done working.
148134 SmallVector DeadInsts;
174160 AU.addRequired();
175161 AU.addPreserved();
176162 }
177
178 /// getCastedVersionOf - Return the specified value casted to uintptr_t.
179 ///
180 Value *getCastedVersionOf(Instruction::CastOps opcode, Value *V);
163
181164 private:
182165 bool AddUsersIfInteresting(Instruction *I, Loop *L,
183166 SmallPtrSet &Processed);
184 SCEVHandle GetExpressionSCEV(Instruction *E);
185167 ICmpInst *ChangeCompareStride(Loop *L, ICmpInst *Cond,
186168 IVStrideUse* &CondUse,
187169 const SCEVHandle* &CondStride);
248230 return new LoopStrengthReduce(TLI);
249231 }
250232
251 /// getCastedVersionOf - Return the specified value casted to uintptr_t. This
252 /// assumes that the Value* V is of integer or pointer type only.
253 ///
254 Value *LoopStrengthReduce::getCastedVersionOf(Instruction::CastOps opcode,
255 Value *V) {
256 if (V->getType() == UIntPtrTy) return V;
257 if (Constant *CB = dyn_cast(V))
258 return ConstantExpr::getCast(opcode, CB, UIntPtrTy);
259
260 Value *&New = CastedPointers[V];
261 if (New) return New;
262
263 New = SCEVExpander::InsertCastOfTo(opcode, V, UIntPtrTy);
264 DeadInsts.push_back(cast(New));
265 return New;
266 }
267
268
269233 /// DeleteTriviallyDeadInstructions - If any of the instructions is the
270234 /// specified set are trivially dead, delete them and see if this makes any of
271235 /// their operands subsequently dead.
305269 I->eraseFromParent();
306270 Changed = true;
307271 }
308 }
309
310
311 /// GetExpressionSCEV - Compute and return the SCEV for the specified
312 /// instruction.
313 SCEVHandle LoopStrengthReduce::GetExpressionSCEV(Instruction *Exp) {
314 // Pointer to pointer bitcast instructions return the same value as their
315 // operand.
316 if (BitCastInst *BCI = dyn_cast(Exp)) {
317 if (SE->hasSCEV(BCI) || !isa(BCI->getOperand(0)))
318 return SE->getSCEV(BCI);
319 SCEVHandle R = GetExpressionSCEV(cast(BCI->getOperand(0)));
320 SE->setSCEV(BCI, R);
321 return R;
322 }
323
324 // Scalar Evolutions doesn't know how to compute SCEV's for GEP instructions.
325 // If this is a GEP that SE doesn't know about, compute it now and insert it.
326 // If this is not a GEP, or if we have already done this computation, just let
327 // SE figure it out.
328 GetElementPtrInst *GEP = dyn_cast(Exp);
329 if (!GEP || SE->hasSCEV(GEP))
330 return SE->getSCEV(Exp);
331
332 // Analyze all of the subscripts of this getelementptr instruction, looking
333 // for uses that are determined by the trip count of the loop. First, skip
334 // all operands the are not dependent on the IV.
335
336 // Build up the base expression. Insert an LLVM cast of the pointer to
337 // uintptr_t first.
338 SCEVHandle GEPVal = SE->getUnknown(
339 getCastedVersionOf(Instruction::PtrToInt, GEP->getOperand(0)));
340
341 gep_type_iterator GTI = gep_type_begin(GEP);
342
343 for (User::op_iterator i = GEP->op_begin() + 1, e = GEP->op_end();
344 i != e; ++i, ++GTI) {
345 // If this is a use of a recurrence that we can analyze, and it comes before
346 // Op does in the GEP operand list, we will handle this when we process this
347 // operand.
348 if (const StructType *STy = dyn_cast(*GTI)) {
349 const StructLayout *SL = TD->getStructLayout(STy);
350 unsigned Idx = cast(*i)->getZExtValue();
351 uint64_t Offset = SL->getElementOffset(Idx);
352 GEPVal = SE->getAddExpr(GEPVal,
353 SE->getIntegerSCEV(Offset, UIntPtrTy));
354 } else {
355 unsigned GEPOpiBits =
356 (*i)->getType()->getPrimitiveSizeInBits();
357 unsigned IntPtrBits = UIntPtrTy->getPrimitiveSizeInBits();
358 Instruction::CastOps opcode = (GEPOpiBits < IntPtrBits ?
359 Instruction::SExt : (GEPOpiBits > IntPtrBits ? Instruction::Trunc :
360 Instruction::BitCast));
361 Value *OpVal = getCastedVersionOf(opcode, *i);
362 SCEVHandle Idx = SE->getSCEV(OpVal);
363
364 uint64_t TypeSize = TD->getTypePaddedSize(GTI.getIndexedType());
365 if (TypeSize != 1)
366 Idx = SE->getMulExpr(Idx,
367 SE->getConstant(ConstantInt::get(UIntPtrTy,
368 TypeSize)));
369 GEPVal = SE->getAddExpr(GEPVal, Idx);
370 }
371 }
372
373 SE->setSCEV(GEP, GEPVal);
374 GEPlist.push_back(GEP);
375 return GEPVal;
376272 }
377273
378274 /// containsAddRecFromDifferentLoop - Determine whether expression S involves a
601497 return true; // Instruction already handled.
602498
603499 // Get the symbolic expression for this instruction.
604 SCEVHandle ISE = GetExpressionSCEV(I);
500 SCEVHandle ISE = SE->getSCEV(I);
605501 if (isa(ISE)) return false;
606502
607503 // Get the start and stride for this expression.
721617 SmallVectorImpl &DeadInsts);
722618
723619 Value *InsertCodeForBaseAtPosition(const SCEVHandle &NewBase,
620 const Type *Ty,
724621 SCEVExpander &Rewriter,
725622 Instruction *IP, Loop *L);
726623 void dump() const;
734631 }
735632
736633 Value *BasedUser::InsertCodeForBaseAtPosition(const SCEVHandle &NewBase,
634 const Type *Ty,
737635 SCEVExpander &Rewriter,
738636 Instruction *IP, Loop *L) {
739637 // Figure out where we *really* want to insert this code. In particular, if
754652 InsertLoop = InsertLoop->getParentLoop();
755653 }
756654
757 Value *Base = Rewriter.expandCodeFor(NewBase, BaseInsertPt);
655 Value *Base = Rewriter.expandCodeFor(NewBase, Ty, BaseInsertPt);
758656
759657 // If there is no immediate value, skip the next part.
760658 if (Imm->isZero())
767665
768666 // Always emit the immediate (if non-zero) into the same block as the user.
769667 SCEVHandle NewValSCEV = SE->getAddExpr(SE->getUnknown(Base), Imm);
770 return Rewriter.expandCodeFor(NewValSCEV, IP);
771
668 return Rewriter.expandCodeFor(NewValSCEV, Ty, IP);
772669 }
773670
774671
808705 while (isa(InsertPt)) ++InsertPt;
809706 }
810707 }
811 Value *NewVal = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L);
812 // Adjust the type back to match the Inst. Note that we can't use InsertPt
813 // here because the SCEVExpander may have inserted the instructions after
814 // that point, in its efforts to avoid inserting redundant expressions.
815 if (isa(OperandValToReplace->getType())) {
816 NewVal = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr,
817 NewVal,
818 OperandValToReplace->getType());
819 }
708 Value *NewVal = InsertCodeForBaseAtPosition(NewBase,
709 OperandValToReplace->getType(),
710 Rewriter, InsertPt, L);
820711 // Replace the use of the operand Value with the new Phi we just created.
821712 Inst->replaceUsesOfWith(OperandValToReplace, NewVal);
822713
874765 Instruction *InsertPt = (L->contains(OldLoc->getParent())) ?
875766 PN->getIncomingBlock(i)->getTerminator() :
876767 OldLoc->getParent()->getTerminator();
877 Code = InsertCodeForBaseAtPosition(NewBase, Rewriter, InsertPt, L);
878
879 // Adjust the type back to match the PHI. Note that we can't use
880 // InsertPt here because the SCEVExpander may have inserted its
881 // instructions after that point, in its efforts to avoid inserting
882 // redundant expressions.
883 if (isa(PN->getType())) {
884 Code = SCEVExpander::InsertCastOfTo(Instruction::IntToPtr,
885 Code,
886 PN->getType());
887 }
768 Code = InsertCodeForBaseAtPosition(NewBase, PN->getType(),
769 Rewriter, InsertPt, L);
888770
889771 DOUT << " Changing PHI use to ";
890772 DEBUG(WriteAsOperand(*DOUT, Code, /*PrintType=*/false));
920802 }
921803
922804 if (SCEVUnknown *SU = dyn_cast(V))
923 if (ConstantExpr *CE = dyn_cast(SU->getValue()))
924 if (TLI && CE->getOpcode() == Instruction::PtrToInt) {
925 Constant *Op0 = CE->getOperand(0);
926 if (GlobalValue *GV = dyn_cast(Op0)) {
927 TargetLowering::AddrMode AM;
928 AM.BaseGV = GV;
929 AM.HasBaseReg = HasBaseReg;
930 return TLI->isLegalAddressingMode(AM, UseTy);
931 }
932 }
805 if (GlobalValue *GV = dyn_cast(SU->getValue())) {
806 TargetLowering::AddrMode AM;
807 AM.BaseGV = GV;
808 AM.HasBaseReg = HasBaseReg;
809 return TLI->isLegalAddressingMode(AM, UseTy);
810 }
811
933812 return false;
934813 }
935814
15901469 ///
15911470 static PHINode *InsertAffinePhi(SCEVHandle Start, SCEVHandle Step,
15921471 const Loop *L,
1472 const TargetData *TD,
15931473 SCEVExpander &Rewriter) {
15941474 assert(Start->isLoopInvariant(L) && "New PHI start is not loop invariant!");
15951475 assert(Step->isLoopInvariant(L) && "New PHI stride is not loop invariant!");
15971477 BasicBlock *Header = L->getHeader();
15981478 BasicBlock *Preheader = L->getLoopPreheader();
15991479 BasicBlock *LatchBlock = L->getLoopLatch();
1600
1601 PHINode *PN = PHINode::Create(Start->getType(), "lsr.iv", Header->begin());
1602 PN->addIncoming(Rewriter.expandCodeFor(Start, Preheader->getTerminator()),
1480 const Type *Ty = Start->getType();
1481 if (isa(Ty)) Ty = TD->getIntPtrType();
1482
1483 PHINode *PN = PHINode::Create(Ty, "lsr.iv", Header->begin());
1484 PN->addIncoming(Rewriter.expandCodeFor(Start, Ty, Preheader->getTerminator()),
16031485 Preheader);
16041486
16051487 // If the stride is negative, insert a sub instead of an add for the
16111493
16121494 // Insert an add instruction right before the terminator corresponding
16131495 // to the back-edge.
1614 Value *StepV = Rewriter.expandCodeFor(IncAmount, Preheader->getTerminator());
1496 Value *StepV = Rewriter.expandCodeFor(IncAmount, Ty,
1497 Preheader->getTerminator());
16151498 Instruction *IncV;
16161499 if (isNegative) {
16171500 IncV = BinaryOperator::CreateSub(PN, StepV, "lsr.iv.next",
16821565 SCEVHandle Imm = UsersToProcess[i].Imm;
16831566 SCEVHandle Base = UsersToProcess[i].Base;
16841567 SCEVHandle Start = SE->getAddExpr(CommonExprs, Base, Imm);
1685 PHINode *Phi = InsertAffinePhi(Start, Stride, L,
1568 PHINode *Phi = InsertAffinePhi(Start, Stride, L, TD,
16861569 PreheaderRewriter);
16871570 // Loop over all the users with the same base.
16881571 do {
17091592 DOUT << " Inserting new PHI:\n";
17101593
17111594 PHINode *Phi = InsertAffinePhi(SE->getUnknown(CommonBaseV),
1712 Stride, L,
1595 Stride, L, TD,
17131596 PreheaderRewriter);
17141597
17151598 // Remember this in case a later stride is multiple of this.
18151698 bool HaveCommonExprs = !CommonExprs->isZero();
18161699
18171700 const Type *ReplacedTy = CommonExprs->getType();
1701 if (isa(ReplacedTy)) ReplacedTy = TD->getIntPtrType();
18181702
18191703 // If all uses are addresses, consider sinking the immediate part of the
18201704 // common expression back into uses if they can fit in the immediate fields.
18281712 // If the immediate part of the common expression is a GV, check if it's
18291713 // possible to fold it into the target addressing mode.
18301714 GlobalValue *GV = 0;
1831 if (SCEVUnknown *SU = dyn_cast(Imm)) {
1832 if (ConstantExpr *CE = dyn_cast(SU->getValue()))
1833 if (CE->getOpcode() == Instruction::PtrToInt)
1834 GV = dyn_cast(CE->getOperand(0));
1835 }
1715 if (SCEVUnknown *SU = dyn_cast(Imm))
1716 GV = dyn_cast(SU->getValue());
18361717 int64_t Offset = 0;
18371718 if (SCEVConstant *SC = dyn_cast(Imm))
18381719 Offset = SC->getValue()->getSExtValue();
18591740 << *Stride << ":\n"
18601741 << " Common base: " << *CommonExprs << "\n";
18611742
1862 SCEVExpander Rewriter(*SE, *LI);
1863 SCEVExpander PreheaderRewriter(*SE, *LI);
1743 SCEVExpander Rewriter(*SE, *LI, *TD);
1744 SCEVExpander PreheaderRewriter(*SE, *LI, *TD);
18641745
18651746 BasicBlock *Preheader = L->getLoopPreheader();
18661747 Instruction *PreInsertPt = Preheader->getTerminator();
18811762 PreheaderRewriter);
18821763 } else {
18831764 // Emit the initial base value into the loop preheader.
1884 CommonBaseV = PreheaderRewriter.expandCodeFor(CommonExprs, PreInsertPt);
1765 CommonBaseV = PreheaderRewriter.expandCodeFor(CommonExprs, ReplacedTy,
1766 PreInsertPt);
18851767
18861768 // If all uses are addresses, check if it is possible to reuse an IV with a
18871769 // stride that is a factor of this stride. And that the multiple is a number
19091791 Instruction *Inst = UsersToProcess.back().Inst;
19101792
19111793 // Emit the code for Base into the preheader.
1912 Value *BaseV = PreheaderRewriter.expandCodeFor(Base, PreInsertPt);
1913
1914 DOUT << " Examining uses with BASE ";
1915 DEBUG(WriteAsOperand(*DOUT, BaseV, /*PrintType=*/false));
1916 DOUT << ":\n";
1917
1918 // If BaseV is a constant other than 0, make sure that it gets inserted into
1919 // the preheader, instead of being forward substituted into the uses. We do
1920 // this by forcing a BitCast (noop cast) to be inserted into the preheader
1921 // in this case.
1922 if (Constant *C = dyn_cast(BaseV)) {
1923 if (!C->isNullValue() && !fitsInAddressMode(Base, getAccessType(Inst),
1924 TLI, false)) {
1794 Value *BaseV = 0;
1795 if (!Base->isZero()) {
1796 BaseV = PreheaderRewriter.expandCodeFor(Base, Base->getType(),
1797 PreInsertPt);
1798
1799 DOUT << " INSERTING code for BASE = " << *Base << ":";
1800 if (BaseV->hasName())
1801 DOUT << " Result value name = %" << BaseV->getNameStr();
1802 DOUT << "\n";
1803
1804 // If BaseV is a non-zero constant, make sure that it gets inserted into
1805 // the preheader, instead of being forward substituted into the uses. We
1806 // do this by forcing a BitCast (noop cast) to be inserted into the
1807 // preheader in this case.
1808 if (!fitsInAddressMode(Base, getAccessType(Inst), TLI, false)) {
19251809 // We want this constant emitted into the preheader! This is just
19261810 // using cast as a copy so BitCast (no-op cast) is appropriate
19271811 BaseV = new BitCastInst(BaseV, BaseV->getType(), "preheaderinsert",
19521836 User.Inst->moveBefore(LatchBlock->getTerminator());
19531837 }
19541838 if (RewriteOp->getType() != ReplacedTy) {
1955 Instruction::CastOps opcode = Instruction::Trunc;
1956 if (ReplacedTy->getPrimitiveSizeInBits() ==
1957 RewriteOp->getType()->getPrimitiveSizeInBits())
1958 opcode = Instruction::BitCast;
1839 Instruction::CastOps opcode =
1840 CastInst::getCastOpcode(RewriteOp, false, ReplacedTy, false);
1841 assert(opcode != Instruction::SExt &&
1842 opcode != Instruction::ZExt &&
1843 "Unexpected widening cast!");
19591844 RewriteOp = SCEVExpander::InsertCastOfTo(opcode, RewriteOp, ReplacedTy);
19601845 }
19611846
19771862
19781863 // If we are reusing the iv, then it must be multiplied by a constant
19791864 // factor to take advantage of the addressing mode scale component.
1980 if (!isa(RewriteFactor) ||
1981 !cast(RewriteFactor)->isZero()) {
1865 if (!RewriteFactor->isZero()) {
19821866 // If we're reusing an IV with a nonzero base (currently this happens
19831867 // only when all reuses are outside the loop) subtract that base here.
19841868 // The base has been used to initialize the PHI node but we don't want
20091893 // When this use is outside the loop, we earlier subtracted the
20101894 // common base, and are adding it back here. Use the same expression
20111895 // as before, rather than CommonBaseV, so DAGCombiner will zap it.
2012 if (!isa(CommonBaseV) ||
2013 !cast(CommonBaseV)->isZero()) {
1896 if (!CommonExprs->isZero()) {
20141897 if (L->contains(User.Inst->getParent()))
20151898 RewriteExpr = SE->getAddExpr(RewriteExpr,
20161899 SE->getUnknown(CommonBaseV));
20211904
20221905 // Now that we know what we need to do, insert code before User for the
20231906 // immediate and any loop-variant expressions.
2024 if (!isa(BaseV) || !cast(BaseV)->isZero())
1907 if (BaseV)
20251908 // Add BaseV to the PHI value if needed.
20261909 RewriteExpr = SE->getAddExpr(RewriteExpr, SE->getUnknown(BaseV));
20271910
20771960 // e.g.
20781961 // 4, -1, X, 1, 2 ==> 1, -1, 2, 4, X
20791962 struct StrideCompare {
1963 const TargetData *TD;
1964 explicit StrideCompare(const TargetData *td) : TD(td) {}
1965
20801966 bool operator()(const SCEVHandle &LHS, const SCEVHandle &RHS) {
20811967 SCEVConstant *LHSC = dyn_cast(LHS);
20821968 SCEVConstant *RHSC = dyn_cast(RHS);
20951981 // If it's the same value but different type, sort by bit width so
20961982 // that we emit larger induction variables before smaller
20971983 // ones, letting the smaller be re-written in terms of larger ones.
2098 return RHS->getBitWidth() < LHS->getBitWidth();
1984 return TD->getTypeSizeInBits(RHS->getType()) <
1985 TD->getTypeSizeInBits(LHS->getType());
20991986 }
21001987 return LHSC && !RHSC;
21011988 }
21282015
21292016 ICmpInst::Predicate Predicate = Cond->getPredicate();
21302017 int64_t CmpSSInt = SC->getValue()->getSExtValue();
2131 unsigned BitWidth = (*CondStride)->getBitWidth();
2018 unsigned BitWidth = TD->getTypeSizeInBits((*CondStride)->getType());
21322019 uint64_t SignBit = 1ULL << (BitWidth-1);
21332020 const Type *CmpTy = Cond->getOperand(0)->getType();
21342021 const Type *NewCmpTy = NULL;
2135 unsigned TyBits = CmpTy->getPrimitiveSizeInBits();
2022 unsigned TyBits = TD->getTypeSizeInBits(CmpTy);
21362023 unsigned NewTyBits = 0;
21372024 SCEVHandle *NewStride = NULL;
21382025 Value *NewCmpLHS = NULL;
21842071 continue;
21852072
21862073 NewCmpTy = NewCmpLHS->getType();
2187 NewTyBits = isa(NewCmpTy)
2188 ? UIntPtrTy->getPrimitiveSizeInBits()
2189 : NewCmpTy->getPrimitiveSizeInBits();
2074 NewTyBits = TD->getTypeSizeInBits(NewCmpTy);
21902075 if (RequiresTypeConversion(NewCmpTy, CmpTy)) {
21912076 // Check if it is possible to rewrite it using
21922077 // an iv / stride of a smaller integer type.
26032488 #endif
26042489
26052490 // Sort the StrideOrder so we process larger strides first.
2606 std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare());
2491 std::stable_sort(StrideOrder.begin(), StrideOrder.end(), StrideCompare(TD));
26072492
26082493 // Optimize induction variables. Some indvar uses can be transformed to use
26092494 // strides that will be needed for other purposes. A common example of this
26372522 }
26382523
26392524 // We're done analyzing this loop; release all the state we built up for it.
2640 CastedPointers.clear();
26412525 IVUsesByStride.clear();
26422526 IVsByStride.clear();
26432527 StrideOrder.clear();
2644 for (unsigned i=0; i
2645 SE->deleteValueFromRecords(GEPlist[i]);
2646 GEPlist.clear();
26472528
26482529 // Clean up after ourselves
26492530 if (!DeadInsts.empty()) {
0 ; RUN: llvm-as < %s | llc -mtriple=arm-apple-darwin -relocation-model=pic \
1 ; RUN: -mattr=+v6 -stats |& grep asm-printer | grep 41
1 ; RUN: -mattr=+v6 -ifcvt-limit=0 -stats |& grep asm-printer | grep 35
22
33 define void @test(i32 %tmp56222, i32 %tmp36224, i32 %tmp46223, i32 %i.0196.0.ph, i32 %tmp8, i32* %tmp1011, i32** %tmp1, i32* %d2.1.out, i32* %d3.1.out, i32* %d0.1.out, i32* %d1.1.out) {
44 newFuncRoot:
0 ; RUN: llvm-as < %s | llc -march=x86-64 -f -o %t
11 ; RUN: grep inc %t | count 2
22 ; RUN: grep addq %t | count 13
3 ; RUN: grep leaq %t | count 9
3 ; RUN: grep leaq %t | count 8
4 ; RUN: grep leal %t | count 4
45 ; RUN: grep movq %t | count 5
56
67 ; IV users in each of the loops from other loops shouldn't cause LSR
None ; RUN: llvm-as < %s | llc -march=x86 -stats |& grep {Number of reloads omited}
1 ; RUN: llvm-as < %s | llc -march=x86 -stats |& grep {Number of available reloads turned into copies}
0 ; RUN: llvm-as < %s | llc -march=x86 -stats |& grep {Number of reloads omited} | grep 2
1 ; RUN: llvm-as < %s | llc -march=x86 -stats |& not grep {Number of available reloads turned into copies}
2 ; RUN: llvm-as < %s | llc -march=x86 -stats |& grep {Number of machine instrs printed} | grep 39
23 ; PR3495
34
45 target triple = "i386-pc-linux-gnu"
0 ; RUN: llvm-as < %s | llc -march=x86 -relocation-model=static | not grep lea
11 ; RUN: llvm-as < %s | llc -march=x86-64 | not grep lea
22
3 ; _P should be sunk into the loop and folded into the address mode. There
3 ; P should be sunk into the loop and folded into the address mode. There
44 ; shouldn't be any lea instructions inside the loop.
55
66 @B = external global [1000 x i8], align 32