llvm.org GIT mirror llvm / 6ca75fa
[IndVarSimplify] Use control-dependent range information to prove non-negativity This change is motivated by the case when IndVarSimplify doesn't widen a comparison of IV increment because it can't prove IV increment being non-negative. We end up with a redundant trunc of the widened increment on this example. for.body: %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ] %within_limits = icmp ult i32 %i, 64 br i1 %within_limits, label %continue, label %for.end continue: %i.i64 = zext i32 %i to i64 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64 %val = load i32, i32* %arrayidx, align 4 br label %for.inc for.inc: %i.inc = add nsw nuw i32 %i, 1 %cmp = icmp slt i32 %i.inc, %limit br i1 %cmp, label %for.body, label %for.end There is a range check inside of the loop which guarantees the IV to be non-negative. NSW on the increment guarantees that the increment is also non-negative. Teach IndVarSimplify to use the range check to prove non-negativity of loop increments. Reviewed By: sanjoy Differential Revision: https://reviews.llvm.org/D25738 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@284629 91177308-0d34-0410-b5e6-96231b3b80d8 Artur Pilipenko 2 years ago
2 changed file(s) with 323 addition(s) and 2 deletion(s). Raw diff Collapse all Expand all
7979 "only replace exit value when the cost is cheap"),
8080 clEnumValN(AlwaysRepl, "always",
8181 "always replace exit value whenever possible")));
82
83 static cl::opt UsePostIncrementRanges(
84 "indvars-post-increment-ranges", cl::Hidden,
85 cl::desc("Use post increment control-dependent ranges in IndVarSimplify"),
86 cl::init(true));
8287
8388 namespace {
8489 struct RewritePhi;
902907 // Value: the kind of extension used to widen this Instruction.
903908 DenseMap, ExtendKind> ExtendKindMap;
904909
910 typedef std::pair, AssertingVH> DefUserPair;
911 // A map with control-dependent ranges for post increment IV uses. The key is
912 // a pair of IV def and a use of this def denoting the context. The value is
913 // a ConstantRange representing possible values of the def at the given
914 // context.
915 DenseMap PostIncRangeInfos;
916
917 Optional getPostIncRangeInfo(Value *Def,
918 Instruction *UseI) {
919 DefUserPair Key(Def, UseI);
920 auto It = PostIncRangeInfos.find(Key);
921 return It == PostIncRangeInfos.end()
922 ? Optional(None)
923 : Optional(It->second);
924 }
925
926 void calculatePostIncRanges(PHINode *OrigPhi);
927 void calculatePostIncRange(Instruction *NarrowDef, Instruction *NarrowUser);
928 void updatePostIncRangeInfo(Value *Def, Instruction *UseI, ConstantRange R) {
929 DefUserPair Key(Def, UseI);
930 auto It = PostIncRangeInfos.find(Key);
931 if (It == PostIncRangeInfos.end())
932 PostIncRangeInfos.insert({Key, R});
933 else
934 It->second = R.intersectWith(It->second);
935 }
936
905937 public:
906938 WidenIV(const WideIVInfo &WI, LoopInfo *LInfo,
907939 ScalarEvolution *SEv, DominatorTree *DTree,
14281460 ///
14291461 void WidenIV::pushNarrowIVUsers(Instruction *NarrowDef, Instruction *WideDef) {
14301462 const SCEV *NarrowSCEV = SE->getSCEV(NarrowDef);
1431 bool NeverNegative =
1463 bool NonNegativeDef =
14321464 SE->isKnownPredicate(ICmpInst::ICMP_SGE, NarrowSCEV,
14331465 SE->getConstant(NarrowSCEV->getType(), 0));
14341466 for (User *U : NarrowDef->users()) {
14381470 if (!Widened.insert(NarrowUser).second)
14391471 continue;
14401472
1441 NarrowIVUsers.emplace_back(NarrowDef, NarrowUser, WideDef, NeverNegative);
1473 bool NonNegativeUse = false;
1474 if (!NonNegativeDef) {
1475 // We might have a control-dependent range information for this context.
1476 if (auto RangeInfo = getPostIncRangeInfo(NarrowDef, NarrowUser))
1477 NonNegativeUse = RangeInfo->getSignedMin().isNonNegative();
1478 }
1479
1480 NarrowIVUsers.emplace_back(NarrowDef, NarrowUser, WideDef,
1481 NonNegativeDef || NonNegativeUse);
14421482 }
14431483 }
14441484
14781518 SE->properlyDominates(AddRec->getStepRecurrence(*SE), L->getHeader()) &&
14791519 "Loop header phi recurrence inputs do not dominate the loop");
14801520
1521 // Iterate over IV uses (including transitive ones) looking for IV increments
1522 // of the form 'add nsw %iv, '. For each increment and each use of
1523 // the increment calculate control-dependent range information basing on
1524 // dominating conditions inside of the loop (e.g. a range check inside of the
1525 // loop). Calculated ranges are stored in PostIncRangeInfos map.
1526 //
1527 // Control-dependent range information is later used to prove that a narrow
1528 // definition is not negative (see pushNarrowIVUsers). It's difficult to do
1529 // this on demand because when pushNarrowIVUsers needs this information some
1530 // of the dominating conditions might be already widened.
1531 if (UsePostIncrementRanges)
1532 calculatePostIncRanges(OrigPhi);
1533
14811534 // The rewriter provides a value for the desired IV expression. This may
14821535 // either find an existing phi or materialize a new one. Either way, we
14831536 // expect a well-formed cyclic phi-with-increments. i.e. any operand not part
15201573 DeadInsts.emplace_back(DU.NarrowDef);
15211574 }
15221575 return WidePhi;
1576 }
1577
1578 /// Calculates control-dependent range for the given def at the given context
1579 /// by looking at dominating conditions inside of the loop
1580 void WidenIV::calculatePostIncRange(Instruction *NarrowDef,
1581 Instruction *NarrowUser) {
1582 using namespace llvm::PatternMatch;
1583
1584 Value *NarrowDefLHS;
1585 const APInt *NarrowDefRHS;
1586 if (!match(NarrowDef, m_NSWAdd(m_Value(NarrowDefLHS),
1587 m_APInt(NarrowDefRHS))) ||
1588 !NarrowDefRHS->isNonNegative())
1589 return;
1590
1591 auto UpdateRangeFromCondition = [&] (Value *Condition,
1592 bool TrueDest) {
1593 CmpInst::Predicate Pred;
1594 Value *CmpRHS;
1595 if (!match(Condition, m_ICmp(Pred, m_Specific(NarrowDefLHS),
1596 m_Value(CmpRHS))))
1597 return;
1598
1599 CmpInst::Predicate P =
1600 TrueDest ? Pred : CmpInst::getInversePredicate(Pred);
1601
1602 auto CmpRHSRange = SE->getSignedRange(SE->getSCEV(CmpRHS));
1603 auto CmpConstrainedLHSRange =
1604 ConstantRange::makeAllowedICmpRegion(P, CmpRHSRange);
1605 auto NarrowDefRange =
1606 CmpConstrainedLHSRange.addWithNoSignedWrap(*NarrowDefRHS);
1607
1608 updatePostIncRangeInfo(NarrowDef, NarrowUser, NarrowDefRange);
1609 };
1610
1611 BasicBlock *NarrowUserBB = NarrowUser->getParent();
1612 // If NarrowUserBB is statically unreachable asking dominator queries may
1613 // yield suprising results. (e.g. the block may not have a dom tree node)
1614 if (!DT->isReachableFromEntry(NarrowUserBB))
1615 return;
1616
1617 for (auto *DTB = (*DT)[NarrowUserBB]->getIDom();
1618 L->contains(DTB->getBlock());
1619 DTB = DTB->getIDom()) {
1620 auto *BB = DTB->getBlock();
1621 auto *TI = BB->getTerminator();
1622
1623 auto *BI = dyn_cast(TI);
1624 if (!BI || !BI->isConditional())
1625 continue;
1626
1627 auto *TrueSuccessor = BI->getSuccessor(0);
1628 auto *FalseSuccessor = BI->getSuccessor(1);
1629
1630 auto DominatesNarrowUser = [this, NarrowUser] (BasicBlockEdge BBE) {
1631 return BBE.isSingleEdge() &&
1632 DT->dominates(BBE, NarrowUser->getParent());
1633 };
1634
1635 if (DominatesNarrowUser(BasicBlockEdge(BB, TrueSuccessor)))
1636 UpdateRangeFromCondition(BI->getCondition(), /*TrueDest=*/true);
1637
1638 if (DominatesNarrowUser(BasicBlockEdge(BB, FalseSuccessor)))
1639 UpdateRangeFromCondition(BI->getCondition(), /*TrueDest=*/false);
1640 }
1641 }
1642
1643 /// Calculates PostIncRangeInfos map for the given IV
1644 void WidenIV::calculatePostIncRanges(PHINode *OrigPhi) {
1645 SmallPtrSet Visited;
1646 SmallVector Worklist;
1647 Worklist.push_back(OrigPhi);
1648 Visited.insert(OrigPhi);
1649
1650 while (!Worklist.empty()) {
1651 Instruction *NarrowDef = Worklist.pop_back_val();
1652
1653 for (Use &U : NarrowDef->uses()) {
1654 auto *NarrowUser = cast(U.getUser());
1655
1656 // Don't go looking outside the current loop.
1657 auto *NarrowUserLoop = (*LI)[NarrowUser->getParent()];
1658 if (!NarrowUserLoop || !L->contains(NarrowUserLoop))
1659 continue;
1660
1661 if (!Visited.insert(NarrowUser).second)
1662 continue;
1663
1664 Worklist.push_back(NarrowUser);
1665
1666 calculatePostIncRange(NarrowDef, NarrowUser);
1667 }
1668 }
15231669 }
15241670
15251671 //===----------------------------------------------------------------------===//
0 ; RUN: opt < %s -indvars -indvars-post-increment-ranges -S | FileCheck %s
1
2 target datalayout = "p:64:64:64-n32:64"
3
4 ; When the IV in this loop is widened we want to widen this use as well:
5 ; icmp slt i32 %i.inc, %limit
6 ; In order to do this indvars need to prove that the narrow IV def (%i.inc)
7 ; is not-negative from the range check inside of the loop.
8 define void @test(i32* %base, i32 %limit, i32 %start) {
9 ; CHECK-LABEL: @test(
10 ; CHECK-NOT: trunc
11
12 for.body.lr.ph:
13 br label %for.body
14
15 for.body:
16 %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
17 %within_limits = icmp ult i32 %i, 64
18 br i1 %within_limits, label %continue, label %for.end
19
20 continue:
21 %i.i64 = zext i32 %i to i64
22 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
23 %val = load i32, i32* %arrayidx, align 4
24 br label %for.inc
25
26 for.inc:
27 %i.inc = add nsw nuw i32 %i, 1
28 %cmp = icmp slt i32 %i.inc, %limit
29 br i1 %cmp, label %for.body, label %for.end
30
31 for.end:
32 br label %exit
33
34 exit:
35 ret void
36 }
37
38 define void @test_false_edge(i32* %base, i32 %limit, i32 %start) {
39 ; CHECK-LABEL: @test_false_edge(
40 ; CHECK-NOT: trunc
41
42 for.body.lr.ph:
43 br label %for.body
44
45 for.body:
46 %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
47 %out_of_bounds = icmp ugt i32 %i, 64
48 br i1 %out_of_bounds, label %for.end, label %continue
49
50 continue:
51 %i.i64 = zext i32 %i to i64
52 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
53 %val = load i32, i32* %arrayidx, align 4
54 br label %for.inc
55
56 for.inc:
57 %i.inc = add nsw nuw i32 %i, 1
58 %cmp = icmp slt i32 %i.inc, %limit
59 br i1 %cmp, label %for.body, label %for.end
60
61 for.end:
62 br label %exit
63
64 exit:
65 ret void
66 }
67
68 define void @test_range_metadata(i32* %array_length_ptr, i32* %base,
69 i32 %limit, i32 %start) {
70 ; CHECK-LABEL: @test_range_metadata(
71 ; CHECK-NOT: trunc
72
73 for.body.lr.ph:
74 br label %for.body
75
76 for.body:
77 %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
78 %array_length = load i32, i32* %array_length_ptr, !range !{i32 0, i32 64 }
79 %within_limits = icmp ult i32 %i, %array_length
80 br i1 %within_limits, label %continue, label %for.end
81
82 continue:
83 %i.i64 = zext i32 %i to i64
84 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
85 %val = load i32, i32* %arrayidx, align 4
86 br label %for.inc
87
88 for.inc:
89 %i.inc = add nsw nuw i32 %i, 1
90 %cmp = icmp slt i32 %i.inc, %limit
91 br i1 %cmp, label %for.body, label %for.end
92
93 for.end:
94 br label %exit
95
96 exit:
97 ret void
98 }
99
100 ; Negative version of the test above, we don't know anything about
101 ; array_length_ptr range.
102 define void @test_neg(i32* %array_length_ptr, i32* %base,
103 i32 %limit, i32 %start) {
104 ; CHECK-LABEL: @test_neg(
105 ; CHECK: trunc i64
106
107 for.body.lr.ph:
108 br label %for.body
109
110 for.body:
111 %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
112 %array_length = load i32, i32* %array_length_ptr
113 %within_limits = icmp ult i32 %i, %array_length
114 br i1 %within_limits, label %continue, label %for.end
115
116 continue:
117 %i.i64 = zext i32 %i to i64
118 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
119 %val = load i32, i32* %arrayidx, align 4
120 br label %for.inc
121
122 for.inc:
123 %i.inc = add nsw nuw i32 %i, 1
124 %cmp = icmp slt i32 %i.inc, %limit
125 br i1 %cmp, label %for.body, label %for.end
126
127 for.end:
128 br label %exit
129
130 exit:
131 ret void
132 }
133
134 define void @test_transitive_use(i32* %base, i32 %limit, i32 %start) {
135 ; CHECK-LABEL: @test_transitive_use(
136 ; CHECK-NOT: trunc
137 ; CHECK: %result = icmp slt i64
138
139 for.body.lr.ph:
140 br label %for.body
141
142 for.body:
143 %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
144 %within_limits = icmp ult i32 %i, 64
145 br i1 %within_limits, label %continue, label %for.end
146
147 continue:
148 %i.mul.3 = mul nsw nuw i32 %i, 3
149 %mul_within = icmp ult i32 %i.mul.3, 64
150 br i1 %mul_within, label %guarded, label %continue.2
151
152 guarded:
153 %i.mul.3.inc = add nsw nuw i32 %i.mul.3, 1
154 %result = icmp slt i32 %i.mul.3.inc, %limit
155 br i1 %result, label %continue.2, label %for.end
156
157 continue.2:
158 %i.i64 = zext i32 %i to i64
159 %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
160 %val = load i32, i32* %arrayidx, align 4
161 br label %for.inc
162
163 for.inc:
164 %i.inc = add nsw nuw i32 %i, 1
165 %cmp = icmp slt i32 %i.inc, %limit
166 br i1 %cmp, label %for.body, label %for.end
167
168
169 for.end:
170 br label %exit
171
172 exit:
173 ret void
174 }