llvm.org GIT mirror llvm / c570191
LSR should only reuse phis that match its formula. Fixes rdar://problem/5064068 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@141442 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 8 years ago
3 changed file(s) with 129 addition(s) and 65 deletion(s). Raw diff Collapse all Expand all
6363 /// in a more literal form.
6464 bool CanonicalMode;
6565
66 /// When invoked from LSR, the expander is in "strength reduction" mode. The
67 /// only difference is that phi's are only reused if they are already in
68 /// "expanded" form.
69 bool LSRMode;
70
6671 typedef IRBuilder BuilderType;
6772 BuilderType Builder;
6873
7277 /// SCEVExpander - Construct a SCEVExpander in "canonical" mode.
7378 explicit SCEVExpander(ScalarEvolution &se, const char *name)
7479 : SE(se), IVName(name), IVIncInsertLoop(0), IVIncInsertPos(0),
75 CanonicalMode(true), Builder(se.getContext(), TargetFolder(se.TD)) {}
80 CanonicalMode(true), LSRMode(false),
81 Builder(se.getContext(), TargetFolder(se.TD)) {}
7682
7783 /// clear - Erase the contents of the InsertedExpressions map so that users
7884 /// trying to expand the same expression into multiple BasicBlocks or
8793 /// canonical induction variable of the specified type for the specified
8894 /// loop (inserting one if there is none). A canonical induction variable
8995 /// starts at zero and steps by one on each iteration.
90 PHINode *getOrInsertCanonicalInductionVariable(const Loop *L,
91 Type *Ty);
96 PHINode *getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty);
9297
9398 /// expandCodeFor - Insert code to directly compute the specified SCEV
9499 /// expression into the program. The inserted code is inserted into the
126131 /// is useful for late optimization passes.
127132 void disableCanonicalMode() { CanonicalMode = false; }
128133
134 void enableLSRMode() { LSRMode = true; }
135
129136 /// clearInsertPoint - Clear the current insertion point. This is useful
130137 /// if the instruction that had been serving as the insertion point may
131138 /// have been deleted.
132139 void clearInsertPoint() {
133140 Builder.ClearInsertionPoint();
134141 }
135
136142 private:
137143 LLVMContext &getContext() const { return SE.getContext(); }
138144
207213
208214 void restoreInsertPoint(BasicBlock *BB, BasicBlock::iterator I);
209215
216 bool isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
217
218 bool isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L,
219 Type *ExpandTy);
220
210221 Value *expandAddRecExprLiterally(const SCEVAddRecExpr *);
211222 PHINode *getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
212223 const Loop *L,
1616 #include "llvm/Analysis/LoopInfo.h"
1717 #include "llvm/IntrinsicInst.h"
1818 #include "llvm/LLVMContext.h"
19 #include "llvm/Support/Debug.h"
1920 #include "llvm/Target/TargetData.h"
2021 #include "llvm/ADT/STLExtras.h"
2122
842843 }
843844 }
844845
846 /// Determine if this is a well-behaved chain of instructions leading back to
847 /// the PHI. If so, it may be reused by expanded expressions.
848 bool SCEVExpander::isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV,
849 const Loop *L) {
850 if (IncV->getNumOperands() == 0 || isa(IncV) ||
851 (isa(IncV) && !isa(IncV)))
852 return false;
853 // If any of the operands don't dominate the insert position, bail.
854 // Addrec operands are always loop-invariant, so this can only happen
855 // if there are instructions which haven't been hoisted.
856 if (L == IVIncInsertLoop) {
857 for (User::op_iterator OI = IncV->op_begin()+1,
858 OE = IncV->op_end(); OI != OE; ++OI)
859 if (Instruction *OInst = dyn_cast(OI))
860 if (!SE.DT->dominates(OInst, IVIncInsertPos))
861 return false;
862 }
863 // Advance to the next instruction.
864 IncV = dyn_cast(IncV->getOperand(0));
865 if (!IncV)
866 return false;
867
868 if (IncV->mayHaveSideEffects())
869 return false;
870
871 if (IncV != PN)
872 return true;
873
874 return isNormalAddRecExprPHI(PN, IncV, L);
875 }
876
877 /// Determine if this cyclic phi is in a form that would have been generated by
878 /// LSR. We don't care if the phi was actually expanded in this pass, as long
879 /// as it is in a low-cost form, for example, no implied multiplication. This
880 /// should match any patterns generated by getAddRecExprPHILiterally and
881 /// expandAddtoGEP.
882 bool SCEVExpander::isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV,
883 const Loop *L, Type *ExpandTy) {
884 switch (IncV->getOpcode()) {
885 // Check for a simple Add/Sub or GEP of a loop invariant step.
886 case Instruction::Add:
887 case Instruction::Sub:
888 return IncV->getOperand(0) == PN
889 && L->isLoopInvariant(IncV->getOperand(1));
890 case Instruction::BitCast:
891 IncV = dyn_cast(IncV->getOperand(0));
892 if (!IncV)
893 return false;
894 // fall-thru to GEP handling
895 case Instruction::GetElementPtr: {
896 // This must be a pointer addition of constants (pretty) or some number of
897 // address-size elements (ugly).
898 for (Instruction::op_iterator I = IncV->op_begin()+1, E = IncV->op_end();
899 I != E; ++I) {
900 if (isa(*I))
901 continue;
902 // ugly geps have 2 operands.
903 // i1* is used by the expander to represent an address-size element.
904 if (IncV->getNumOperands() != 2)
905 return false;
906 unsigned AS = cast(ExpandTy)->getAddressSpace();
907 if (IncV->getType() != Type::getInt1PtrTy(SE.getContext(), AS)
908 && IncV->getType() != Type::getInt8PtrTy(SE.getContext(), AS))
909 return false;
910 break;
911 }
912 IncV = dyn_cast(IncV->getOperand(0));
913 if (IncV && IncV->getOpcode() == Instruction::BitCast)
914 IncV = dyn_cast(IncV->getOperand(0));
915 return IncV == PN;
916 }
917 default:
918 return false;
919 }
920 }
921
845922 /// getAddRecExprPHILiterally - Helper for expandAddRecExprLiterally. Expand
846923 /// the base addrec, which is the addrec without any non-loop-dominating
847924 /// values, and return the PHI.
853930 assert((!IVIncInsertLoop||IVIncInsertPos) && "Uninitialized insert position");
854931
855932 // Reuse a previously-inserted PHI, if present.
856 for (BasicBlock::iterator I = L->getHeader()->begin();
857 PHINode *PN = dyn_cast(I); ++I)
858 if (SE.isSCEVable(PN->getType()) &&
859 (SE.getEffectiveSCEVType(PN->getType()) ==
860 SE.getEffectiveSCEVType(Normalized->getType())) &&
861 SE.getSCEV(PN) == Normalized)
862 if (BasicBlock *LatchBlock = L->getLoopLatch()) {
863 Instruction *IncV =
864 cast(PN->getIncomingValueForBlock(LatchBlock));
865
866 // Determine if this is a well-behaved chain of instructions leading
867 // back to the PHI. It probably will be, if we're scanning an inner
868 // loop already visited by LSR for example, but it wouldn't have
869 // to be.
933 BasicBlock *LatchBlock = L->getLoopLatch();
934 if (LatchBlock) {
935 for (BasicBlock::iterator I = L->getHeader()->begin();
936 PHINode *PN = dyn_cast(I); ++I) {
937 if (!SE.isSCEVable(PN->getType()) ||
938 (SE.getEffectiveSCEVType(PN->getType()) !=
939 SE.getEffectiveSCEVType(Normalized->getType())) ||
940 SE.getSCEV(PN) != Normalized)
941 continue;
942
943 Instruction *IncV =
944 cast(PN->getIncomingValueForBlock(LatchBlock));
945
946 if (LSRMode) {
947 if (!isExpandedAddRecExprPHI(PN, IncV, L, ExpandTy))
948 continue;
949 }
950 else {
951 if (!isNormalAddRecExprPHI(PN, IncV, L))
952 continue;
953 }
954 // Ok, the add recurrence looks usable.
955 // Remember this PHI, even in post-inc mode.
956 InsertedValues.insert(PN);
957 // Remember the increment.
958 rememberInstruction(IncV);
959 if (L == IVIncInsertLoop)
870960 do {
871 if (IncV->getNumOperands() == 0 || isa(IncV) ||
872 (isa(IncV) && !isa(IncV))) {
873 IncV = 0;
961 if (SE.DT->dominates(IncV, IVIncInsertPos))
874962 break;
875 }
876 // If any of the operands don't dominate the insert position, bail.
877 // Addrec operands are always loop-invariant, so this can only happen
878 // if there are instructions which haven't been hoisted.
879 if (L == IVIncInsertLoop) {
880 for (User::op_iterator OI = IncV->op_begin()+1,
881 OE = IncV->op_end(); OI != OE; ++OI)
882 if (Instruction *OInst = dyn_cast(OI))
883 if (!SE.DT->dominates(OInst, IVIncInsertPos)) {
884 IncV = 0;
885 break;
886 }
887 }
888 if (!IncV)
889 break;
890 // Advance to the next instruction.
891 IncV = dyn_cast(IncV->getOperand(0));
892 if (!IncV)
893 break;
894 if (IncV->mayHaveSideEffects()) {
895 IncV = 0;
896 break;
897 }
963 // Make sure the increment is where we want it. But don't move it
964 // down past a potential existing post-inc user.
965 IncV->moveBefore(IVIncInsertPos);
966 IVIncInsertPos = IncV;
967 IncV = cast(IncV->getOperand(0));
898968 } while (IncV != PN);
899
900 if (IncV) {
901 // Ok, the add recurrence looks usable.
902 // Remember this PHI, even in post-inc mode.
903 InsertedValues.insert(PN);
904 // Remember the increment.
905 IncV = cast(PN->getIncomingValueForBlock(LatchBlock));
906 rememberInstruction(IncV);
907 if (L == IVIncInsertLoop)
908 do {
909 if (SE.DT->dominates(IncV, IVIncInsertPos))
910 break;
911 // Make sure the increment is where we want it. But don't move it
912 // down past a potential existing post-inc user.
913 IncV->moveBefore(IVIncInsertPos);
914 IVIncInsertPos = IncV;
915 IncV = cast(IncV->getOperand(0));
916 } while (IncV != PN);
917 return PN;
918 }
919 }
969 return PN;
970 }
971 }
920972
921973 // Save the original insertion point so we can restore it when we're done.
922974 BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
37693769
37703770 SCEVExpander Rewriter(SE, "lsr");
37713771 Rewriter.disableCanonicalMode();
3772 Rewriter.enableLSRMode();
37723773 Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
37733774
37743775 // Expand the new value definitions and update the users.