llvm.org GIT mirror llvm / 2fabd46
indvars -disable-iv-rewrite: Adds support for eliminating identity ops. This is a rewrite of the IV simplification algorithm used by -disable-iv-rewrite. To avoid perturbing the default mode, I temporarily split the driver and created SimplifyIVUsersNoRewrite. The idea is to avoid doing opcode/pattern matching inside IndVarSimplify. SCEV already does it. We want to optimize with the full generality of SCEV, but optimize def-use chains top down on-demand rather than rewriting the entire expression bottom-up. This was easy to do for operations that SCEV can prove are identity function. So we're now eliminating bitmasks and zero extends this way. A result of this rewrite is that indvars -disable-iv-rewrite no longer requires IVUsers. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@133502 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 8 years ago
2 changed file(s) with 242 addition(s) and 104 deletion(s). Raw diff Collapse all Expand all
6161 #include "llvm/ADT/STLExtras.h"
6262 using namespace llvm;
6363
64 STATISTIC(NumRemoved , "Number of aux indvars removed");
65 STATISTIC(NumWidened , "Number of indvars widened");
66 STATISTIC(NumInserted, "Number of canonical indvars added");
67 STATISTIC(NumReplaced, "Number of exit values replaced");
68 STATISTIC(NumLFTR , "Number of loop exit tests replaced");
69 STATISTIC(NumElimExt , "Number of IV sign/zero extends eliminated");
70 STATISTIC(NumElimRem , "Number of IV remainder operations eliminated");
71 STATISTIC(NumElimCmp , "Number of IV comparisons eliminated");
64 STATISTIC(NumRemoved , "Number of aux indvars removed");
65 STATISTIC(NumWidened , "Number of indvars widened");
66 STATISTIC(NumInserted , "Number of canonical indvars added");
67 STATISTIC(NumReplaced , "Number of exit values replaced");
68 STATISTIC(NumLFTR , "Number of loop exit tests replaced");
69 STATISTIC(NumElimIdentity, "Number of IV identities eliminated");
70 STATISTIC(NumElimExt , "Number of IV sign/zero extends eliminated");
71 STATISTIC(NumElimRem , "Number of IV remainder operations eliminated");
72 STATISTIC(NumElimCmp , "Number of IV comparisons eliminated");
7273
7374 // DisableIVRewrite mode currently affects IVUsers, so is defined in libAnalysis
7475 // and referenced here.
8384 ScalarEvolution *SE;
8485 DominatorTree *DT;
8586 TargetData *TD;
87
88 PHINode *CurrIV; // Current IV being simplified.
89
90 // Instructions processed by SimplifyIVUsers for CurrIV.
91 SmallPtrSet Simplified;
92
93 // Use-def pairs if IVUsers waiting to be processed for CurrIV.
94 SmallVector, 8> SimpleIVUsers;
95
8696 SmallVector DeadInsts;
8797 bool Changed;
8898 public:
8999
90100 static char ID; // Pass identification, replacement for typeid
91 IndVarSimplify() : LoopPass(ID), IU(0), LI(0), SE(0), DT(0), TD(0) {
101 IndVarSimplify() : LoopPass(ID), IU(0), LI(0), SE(0), DT(0), TD(0),
102 CurrIV(0), Changed(false) {
92103 initializeIndVarSimplifyPass(*PassRegistry::getPassRegistry());
93104 }
94105
104115 AU.addPreserved();
105116 AU.addPreservedID(LoopSimplifyID);
106117 AU.addPreservedID(LCSSAID);
107 AU.addPreserved();
118 if (!DisableIVRewrite)
119 AU.addPreserved();
108120 AU.setPreservesCFG();
109121 }
110122
112124 bool isValidRewrite(Value *FromVal, Value *ToVal);
113125
114126 void SimplifyIVUsers(SCEVExpander &Rewriter);
127 void SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter);
128
129 bool EliminateIVUser(Instruction *UseInst, Instruction *IVOperand);
115130 void EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand);
116131 void EliminateIVRemainder(BinaryOperator *Rem,
117132 Value *IVOperand,
118133 bool IsSigned,
119134 PHINode *IVPhi);
135 void pushIVUsers(Instruction *Def);
136 bool isSimpleIVUser(Instruction *I, const Loop *L);
120137 void RewriteNonIntegerIVs(Loop *L);
121138
122139 ICmpInst *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount,
482499 SE->forgetLoop(L);
483500 }
484501
502 /// SimplifyIVUsers - Iteratively perform simplification on IVUsers within this
503 /// loop. IVUsers is treated as a worklist. Each successive simplification may
504 /// push more users which may themselves be candidates for simplification.
505 ///
506 /// This is the old approach to IV simplification to be replaced by
507 /// SimplifyIVUsersNoRewrite.
508 ///
509 void IndVarSimplify::SimplifyIVUsers(SCEVExpander &Rewriter) {
510 // Each round of simplification involves a round of eliminating operations
511 // followed by a round of widening IVs. A single IVUsers worklist is used
512 // across all rounds. The inner loop advances the user. If widening exposes
513 // more uses, then another pass through the outer loop is triggered.
514 for (IVUsers::iterator I = IU->begin(); I != IU->end(); ++I) {
515 Instruction *UseInst = I->getUser();
516 Value *IVOperand = I->getOperandValToReplace();
517
518 if (ICmpInst *ICmp = dyn_cast(UseInst)) {
519 EliminateIVComparison(ICmp, IVOperand);
520 continue;
521 }
522 if (BinaryOperator *Rem = dyn_cast(UseInst)) {
523 bool IsSigned = Rem->getOpcode() == Instruction::SRem;
524 if (IsSigned || Rem->getOpcode() == Instruction::URem) {
525 EliminateIVRemainder(Rem, IVOperand, IsSigned, I->getPhi());
526 continue;
527 }
528 }
529 }
530 }
531
485532 namespace {
486533 // Collect information about induction variables that are used by sign/zero
487534 // extend operations. This information is recorded by CollectExtend and
492539
493540 WideIVInfo() : WidestNativeType(0), IsSigned(false) {}
494541 };
495 typedef std::map WideIVMap;
496542 }
497543
498544 /// CollectExtend - Update information about the induction variable that is
499545 /// extended by this sign or zero extend operation. This is used to determine
500546 /// the final width of the IV before actually widening it.
501 static void CollectExtend(CastInst *Cast, PHINode *Phi, bool IsSigned,
502 WideIVMap &IVMap, ScalarEvolution *SE,
503 const TargetData *TD) {
547 static void CollectExtend(CastInst *Cast, bool IsSigned, WideIVInfo &WI,
548 ScalarEvolution *SE, const TargetData *TD) {
504549 const Type *Ty = Cast->getType();
505550 uint64_t Width = SE->getTypeSizeInBits(Ty);
506551 if (TD && !TD->isLegalInteger(Width))
507552 return;
508553
509 WideIVInfo &IVInfo = IVMap[Phi];
510 if (!IVInfo.WidestNativeType) {
511 IVInfo.WidestNativeType = SE->getEffectiveSCEVType(Ty);
512 IVInfo.IsSigned = IsSigned;
554 if (!WI.WidestNativeType) {
555 WI.WidestNativeType = SE->getEffectiveSCEVType(Ty);
556 WI.IsSigned = IsSigned;
513557 return;
514558 }
515559
516560 // We extend the IV to satisfy the sign of its first user, arbitrarily.
517 if (IVInfo.IsSigned != IsSigned)
561 if (WI.IsSigned != IsSigned)
518562 return;
519563
520 if (Width > SE->getTypeSizeInBits(IVInfo.WidestNativeType))
521 IVInfo.WidestNativeType = SE->getEffectiveSCEVType(Ty);
564 if (Width > SE->getTypeSizeInBits(WI.WidestNativeType))
565 WI.WidestNativeType = SE->getEffectiveSCEVType(Ty);
522566 }
523567
524568 namespace {
528572 /// inserting truncs whenever we stop propagating the type.
529573 ///
530574 class WidenIV {
575 // Parameters
531576 PHINode *OrigPhi;
532577 const Type *WideType;
533578 bool IsSigned;
534579
535 IVUsers *IU;
536 LoopInfo *LI;
537 Loop *L;
580 // Context
581 LoopInfo *LI;
582 Loop *L;
538583 ScalarEvolution *SE;
539 DominatorTree *DT;
540 SmallVectorImpl &DeadInsts;
541
584 DominatorTree *DT;
585
586 // Result
542587 PHINode *WidePhi;
543588 Instruction *WideInc;
544589 const SCEV *WideIncExpr;
545
546 SmallPtrSet Processed;
590 SmallVectorImpl &DeadInsts;
591
592 SmallPtrSet Widened;
547593
548594 public:
549 WidenIV(PHINode *PN, const WideIVInfo &IVInfo, IVUsers *IUsers,
550 LoopInfo *LInfo, ScalarEvolution *SEv, DominatorTree *DTree,
595 WidenIV(PHINode *PN, const WideIVInfo &WI, LoopInfo *LInfo,
596 ScalarEvolution *SEv, DominatorTree *DTree,
551597 SmallVectorImpl &DI) :
552598 OrigPhi(PN),
553 WideType(IVInfo.WidestNativeType),
554 IsSigned(IVInfo.IsSigned),
555 IU(IUsers),
599 WideType(WI.WidestNativeType),
600 IsSigned(WI.IsSigned),
556601 LI(LInfo),
557602 L(LI->getLoopFor(OrigPhi->getParent())),
558603 SE(SEv),
559604 DT(DTree),
560 DeadInsts(DI),
561605 WidePhi(0),
562606 WideInc(0),
563 WideIncExpr(0) {
607 WideIncExpr(0),
608 DeadInsts(DI) {
564609 assert(L->getHeader() == OrigPhi->getParent() && "Phi must be an IV");
565610 }
566611
567 bool CreateWideIV(SCEVExpander &Rewriter);
612 PHINode *CreateWideIV(SCEVExpander &Rewriter);
568613
569614 protected:
570615 Instruction *CloneIVUser(Instruction *NarrowUse,
578623 Instruction *WideDef);
579624 };
580625 } // anonymous namespace
581
582 /// SimplifyIVUsers - Iteratively perform simplification on IVUsers within this
583 /// loop. IVUsers is treated as a worklist. Each successive simplification may
584 /// push more users which may themselves be candidates for simplification.
585 ///
586 void IndVarSimplify::SimplifyIVUsers(SCEVExpander &Rewriter) {
587 WideIVMap IVMap;
588
589 // Each round of simplification involves a round of eliminating operations
590 // followed by a round of widening IVs. A single IVUsers worklist is used
591 // across all rounds. The inner loop advances the user. If widening exposes
592 // more uses, then another pass through the outer loop is triggered.
593 for (IVUsers::iterator I = IU->begin(), E = IU->end(); I != E;) {
594 for(; I != E; ++I) {
595 Instruction *UseInst = I->getUser();
596 Value *IVOperand = I->getOperandValToReplace();
597
598 if (DisableIVRewrite) {
599 if (CastInst *Cast = dyn_cast(UseInst)) {
600 bool IsSigned = Cast->getOpcode() == Instruction::SExt;
601 if (IsSigned || Cast->getOpcode() == Instruction::ZExt) {
602 CollectExtend(Cast, I->getPhi(), IsSigned, IVMap, SE, TD);
603 continue;
604 }
605 }
606 }
607 if (ICmpInst *ICmp = dyn_cast(UseInst)) {
608 EliminateIVComparison(ICmp, IVOperand);
609 continue;
610 }
611 if (BinaryOperator *Rem = dyn_cast(UseInst)) {
612 bool IsSigned = Rem->getOpcode() == Instruction::SRem;
613 if (IsSigned || Rem->getOpcode() == Instruction::URem) {
614 EliminateIVRemainder(Rem, IVOperand, IsSigned, I->getPhi());
615 continue;
616 }
617 }
618 }
619 for (WideIVMap::const_iterator I = IVMap.begin(), E = IVMap.end();
620 I != E; ++I) {
621 WidenIV Widener(I->first, I->second, IU, LI, SE, DT, DeadInsts);
622 if (Widener.CreateWideIV(Rewriter))
623 Changed = true;
624 }
625 }
626 }
627626
628627 static Value *getExtend( Value *NarrowOper, const Type *WideType,
629628 bool IsSigned, IRBuilder<> &Builder) {
743742 return 0;
744743
745744 // Handle data flow merges and bizarre phi cycles.
746 if (!Processed.insert(NarrowUse))
745 if (!Widened.insert(NarrowUse))
747746 return 0;
748747
749748 // Our raison d'etre! Eliminate sign and zero extension.
774773 NarrowUse->replaceAllUsesWith(NewDef);
775774 DeadInsts.push_back(NarrowUse);
776775 }
777 // Now that the extend is gone, expose it's uses to IVUsers for potential
778 // further simplification within SimplifyIVUsers.
779 IU->AddUsersIfInteresting(WideDef, WidePhi);
776 // Now that the extend is gone, we want to expose it's uses for potential
777 // further simplification. We don't need to directly inform SimplifyIVUsers
778 // of the new users, because their parent IV will be processed later as a
779 // new loop phi. If we preserved IVUsers analysis, we would also want to
780 // push the uses of WideDef here.
780781
781782 // No further widening is needed. The deceased [sz]ext had done it for us.
782783 return 0;
806807 // outside the loop without overflow. This suggests that the wide use
807808 // evaluates to the same expression as the extended narrow use, but doesn't
808809 // absolutely guarantee it. Hence the following failsafe check. In rare cases
809 // where it fails, we simple throw away the newly created wide use.
810 // where it fails, we simply throw away the newly created wide use.
810811 if (WideAddRec != SE->getSCEV(WideUse)) {
811812 DEBUG(dbgs() << "Wide use expression mismatch: " << *WideUse
812813 << ": " << *SE->getSCEV(WideUse) << " != " << *WideAddRec << "\n");
821822 /// CreateWideIV - Process a single induction variable. First use the
822823 /// SCEVExpander to create a wide induction variable that evaluates to the same
823824 /// recurrence as the original narrow IV. Then use a worklist to forward
824 /// traverse the narrow IV's def-use chain. After WidenIVUse as processed all
825 /// traverse the narrow IV's def-use chain. After WidenIVUse has processed all
825826 /// interesting IV users, the narrow IV will be isolated for removal by
826827 /// DeleteDeadPHIs.
827828 ///
828829 /// It would be simpler to delete uses as they are processed, but we must avoid
829830 /// invalidating SCEV expressions.
830831 ///
831 bool WidenIV::CreateWideIV(SCEVExpander &Rewriter) {
832 PHINode *WidenIV::CreateWideIV(SCEVExpander &Rewriter) {
832833 // Is this phi an induction variable?
833834 const SCEVAddRecExpr *AddRec = dyn_cast(SE->getSCEV(OrigPhi));
834835 if (!AddRec)
835 return false;
836 return NULL;
836837
837838 // Widen the induction variable expression.
838839 const SCEV *WideIVExpr = IsSigned ?
845846 // Can the IV be extended outside the loop without overflow?
846847 AddRec = dyn_cast(WideIVExpr);
847848 if (!AddRec || AddRec->getLoop() != L)
848 return false;
849
850 // An AddRec must have loop-invariant operands. Since this AddRec it
849 return NULL;
850
851 // An AddRec must have loop-invariant operands. Since this AddRec is
851852 // materialized by a loop header phi, the expression cannot have any post-loop
852853 // operands, so they must dominate the loop header.
853854 assert(SE->properlyDominates(AddRec->getStart(), L->getHeader()) &&
875876 ++NumWidened;
876877
877878 // Traverse the def-use chain using a worklist starting at the original IV.
878 assert(Processed.empty() && "expect initial state" );
879 assert(Widened.empty() && "expect initial state" );
879880
880881 // Each worklist entry has a Narrow def-use link and Wide def.
881882 SmallVector, 8> NarrowIVUsers;
905906 if (NarrowDef->use_empty())
906907 DeadInsts.push_back(NarrowDef);
907908 }
908 return true;
909 return WidePhi;
909910 }
910911
911912 void IndVarSimplify::EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) {
988989 }
989990
990991 // Inform IVUsers about the new users.
991 if (Instruction *I = dyn_cast(Rem->getOperand(0)))
992 IU->AddUsersIfInteresting(I, IVPhi);
993
992 if (IU) {
993 if (Instruction *I = dyn_cast(Rem->getOperand(0)))
994 IU->AddUsersIfInteresting(I, IVPhi);
995 }
994996 DEBUG(dbgs() << "INDVARS: Simplified rem: " << *Rem << '\n');
995997 ++NumElimRem;
996998 Changed = true;
997999 DeadInsts.push_back(Rem);
1000 }
1001
1002 /// EliminateIVUser - Eliminate an operation that consumes a simple IV and has
1003 /// no observable side-effect given the range of IV values.
1004 bool IndVarSimplify::EliminateIVUser(Instruction *UseInst,
1005 Instruction *IVOperand) {
1006 if (ICmpInst *ICmp = dyn_cast(UseInst)) {
1007 EliminateIVComparison(ICmp, IVOperand);
1008 return true;
1009 }
1010 if (BinaryOperator *Rem = dyn_cast(UseInst)) {
1011 bool IsSigned = Rem->getOpcode() == Instruction::SRem;
1012 if (IsSigned || Rem->getOpcode() == Instruction::URem) {
1013 EliminateIVRemainder(Rem, IVOperand, IsSigned, CurrIV);
1014 return true;
1015 }
1016 }
1017
1018 // Eliminate any operation that SCEV can prove is an identity function.
1019 if (!SE->isSCEVable(UseInst->getType()) ||
1020 (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand)))
1021 return false;
1022
1023 UseInst->replaceAllUsesWith(IVOperand);
1024
1025 DEBUG(dbgs() << "INDVARS: Eliminated identity: " << *UseInst << '\n');
1026 ++NumElimIdentity;
1027 Changed = true;
1028 DeadInsts.push_back(UseInst);
1029 return true;
1030 }
1031
1032 /// pushIVUsers - Add all uses of Def to the current IV's worklist.
1033 ///
1034 void IndVarSimplify::pushIVUsers(Instruction *Def) {
1035
1036 for (Value::use_iterator UI = Def->use_begin(), E = Def->use_end();
1037 UI != E; ++UI) {
1038 Instruction *User = cast(*UI);
1039
1040 // Avoid infinite or exponential worklist processing.
1041 // Also ensure unique worklist users.
1042 if (Simplified.insert(User))
1043 SimpleIVUsers.push_back(std::make_pair(User, Def));
1044 }
1045 }
1046
1047 /// isSimpleIVUser - Return true if this instruction generates a simple SCEV
1048 /// expression in terms of that IV.
1049 ///
1050 /// This is similar to IVUsers' isInsteresting() but processes each instruction
1051 /// non-recursively when the operand is already known to be a simpleIVUser.
1052 ///
1053 bool IndVarSimplify::isSimpleIVUser(Instruction *I, const Loop *L) {
1054 if (!SE->isSCEVable(I->getType()))
1055 return false;
1056
1057 // Get the symbolic expression for this instruction.
1058 const SCEV *S = SE->getSCEV(I);
1059
1060 // Only consider affine recurrences.
1061 const SCEVAddRecExpr *AR = dyn_cast(S);
1062 if (AR && AR->getLoop() == L)
1063 return true;
1064
1065 return false;
1066 }
1067
1068 /// SimplifyIVUsersNoRewrite - Iteratively perform simplification on a worklist
1069 /// of IV users. Each successive simplification may push more users which may
1070 /// themselves be candidates for simplification.
1071 ///
1072 /// The "NoRewrite" algorithm does not require IVUsers analysis. Instead, it
1073 /// simplifies instructions in-place during analysis. Rather than rewriting
1074 /// induction variables bottom-up from their users, it transforms a chain of
1075 /// IVUsers top-down, updating the IR only when it encouters a clear
1076 /// optimization opportunitiy. A SCEVExpander "Rewriter" instance is still
1077 /// needed, but only used to generate a new IV (phi) of wider type for sign/zero
1078 /// extend elimination.
1079 ///
1080 /// Once DisableIVRewrite is default, LSR will be the only client of IVUsers.
1081 ///
1082 void IndVarSimplify::SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter) {
1083 // Simplification is performed independently for each IV, as represented by a
1084 // loop header phi. Each round of simplification first iterates through the
1085 // SimplifyIVUsers worklist, then determines whether the current IV should be
1086 // widened. Widening adds a new phi to LoopPhis, inducing another round of
1087 // simplification on the wide IV.
1088 SmallVector LoopPhis;
1089 for (BasicBlock::iterator I = L->getHeader()->begin(); isa(I); ++I) {
1090 LoopPhis.push_back(cast(I));
1091 }
1092 while (!LoopPhis.empty()) {
1093 CurrIV = LoopPhis.pop_back_val();
1094 Simplified.clear();
1095 assert(SimpleIVUsers.empty() && "expect empty IV users list");
1096
1097 WideIVInfo WI;
1098
1099 pushIVUsers(CurrIV);
1100
1101 while (!SimpleIVUsers.empty()) {
1102 Instruction *UseInst, *Operand;
1103 tie(UseInst, Operand) = SimpleIVUsers.pop_back_val();
1104
1105 if (EliminateIVUser(UseInst, Operand)) {
1106 pushIVUsers(Operand);
1107 continue;
1108 }
1109 if (CastInst *Cast = dyn_cast(UseInst)) {
1110 bool IsSigned = Cast->getOpcode() == Instruction::SExt;
1111 if (IsSigned || Cast->getOpcode() == Instruction::ZExt) {
1112 CollectExtend(Cast, IsSigned, WI, SE, TD);
1113 }
1114 continue;
1115 }
1116 if (isSimpleIVUser(UseInst, L)) {
1117 pushIVUsers(UseInst);
1118 }
1119 }
1120 if (WI.WidestNativeType) {
1121 WidenIV Widener(CurrIV, WI, LI, SE, DT, DeadInsts);
1122 if (PHINode *WidePhi = Widener.CreateWideIV(Rewriter)) {
1123 Changed = true;
1124 LoopPhis.push_back(WidePhi);
1125 }
1126 }
1127 }
9981128 }
9991129
10001130 bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
10091139 if (!L->isLoopSimplifyForm())
10101140 return false;
10111141
1012 IU = &getAnalysis();
1142 if (!DisableIVRewrite)
1143 IU = &getAnalysis();
10131144 LI = &getAnalysis();
10141145 SE = &getAnalysis();
10151146 DT = &getAnalysis();
10161147 TD = getAnalysisIfAvailable();
10171148
1149 CurrIV = NULL;
1150 Simplified.clear();
10181151 DeadInsts.clear();
10191152 Changed = false;
10201153
10391172 RewriteLoopExitValues(L, Rewriter);
10401173
10411174 // Eliminate redundant IV users.
1042 SimplifyIVUsers(Rewriter);
1175 if (DisableIVRewrite)
1176 SimplifyIVUsersNoRewrite(L, Rewriter);
1177 else
1178 SimplifyIVUsers(Rewriter);
10431179
10441180 // Compute the type of the largest recurrence expression, and decide whether
10451181 // a canonical induction variable should be inserted.
11451281
11461282 // For completeness, inform IVUsers of the IV use in the newly-created
11471283 // loop exit test instruction.
1148 if (NewICmp)
1284 if (NewICmp && IU)
11491285 IU->AddUsersIfInteresting(cast(NewICmp->getOperand(0)),
11501286 IndVar);
11511287
15781714 }
15791715
15801716 // Add a new IVUsers entry for the newly-created integer PHI.
1581 IU->AddUsersIfInteresting(NewPHI, NewPHI);
1582 }
1717 if (IU)
1718 IU->AddUsersIfInteresting(NewPHI, NewPHI);
1719 }
0 ; RUN: opt < %s -indvars -S | FileCheck %s
1 ; RUN: opt < %s -indvars -disable-iv-rewrite -S | FileCheck %s
12 ; CHECK-NOT: and
23 ; CHECK-NOT: zext
34