llvm.org GIT mirror llvm / 06988bc
Made SCEV's UDiv expressions more canonical. When dividing a recurrence, the initial values low bits can sometimes be ignored. To take advantage of this, added FoldIVUser to IndVarSimplify to fold an IV operand into a udiv/lshr if the operator doesn't affect the result. -indvars -disable-iv-rewrite now transforms i = phi i4 i1 = i0 + 1 idx = i1 >> (2 or more) i4 = i + 4 into i = phi i4 idx = i0 >> ... i4 = i + 4 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@137013 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 8 years ago
3 changed file(s) with 142 addition(s) and 4 deletion(s). Raw diff Collapse all Expand all
20502050 ++MaxShiftAmt;
20512051 IntegerType *ExtTy =
20522052 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
2053 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
20542053 if (const SCEVAddRecExpr *AR = dyn_cast(LHS))
20552054 if (const SCEVConstant *Step =
2056 dyn_cast(AR->getStepRecurrence(*this)))
2057 if (!Step->getValue()->getValue()
2058 .urem(RHSC->getValue()->getValue()) &&
2055 dyn_cast(AR->getStepRecurrence(*this))) {
2056 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
2057 const APInt &StepInt = Step->getValue()->getValue();
2058 const APInt &DivInt = RHSC->getValue()->getValue();
2059 if (!StepInt.urem(DivInt) &&
20592060 getZeroExtendExpr(AR, ExtTy) ==
20602061 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
20612062 getZeroExtendExpr(Step, ExtTy),
20662067 return getAddRecExpr(Operands, AR->getLoop(),
20672068 SCEV::FlagNW);
20682069 }
2070 /// Get a canonical UDivExpr for a recurrence.
2071 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
2072 // We can currently only fold X%N if X is constant.
2073 const SCEVConstant *StartC = dyn_cast(AR->getStart());
2074 if (StartC && !DivInt.urem(StepInt) &&
2075 getZeroExtendExpr(AR, ExtTy) ==
2076 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
2077 getZeroExtendExpr(Step, ExtTy),
2078 AR->getLoop(), SCEV::FlagAnyWrap)) {
2079 const APInt &StartInt = StartC->getValue()->getValue();
2080 const APInt &StartRem = StartInt.urem(StepInt);
2081 if (StartRem != 0)
2082 LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step,
2083 AR->getLoop(), SCEV::FlagNW);
2084 }
2085 }
20692086 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
20702087 if (const SCEVMulExpr *M = dyn_cast(LHS)) {
20712088 SmallVector Operands;
6969 STATISTIC(NumReplaced , "Number of exit values replaced");
7070 STATISTIC(NumLFTR , "Number of loop exit tests replaced");
7171 STATISTIC(NumElimIdentity, "Number of IV identities eliminated");
72 STATISTIC(NumElimOperand, "Number of IV operands folded into a use");
7273 STATISTIC(NumElimExt , "Number of IV sign/zero extends eliminated");
7374 STATISTIC(NumElimRem , "Number of IV remainder operations eliminated");
7475 STATISTIC(NumElimCmp , "Number of IV comparisons eliminated");
140141 void EliminateIVRemainder(BinaryOperator *Rem,
141142 Value *IVOperand,
142143 bool IsSigned);
144
145 bool FoldIVUser(Instruction *UseInst, Instruction *IVOperand);
143146
144147 void SimplifyCongruentIVs(Loop *L);
145148
12971300 return true;
12981301 }
12991302
1303 /// FoldIVUser - Fold an IV operand into its use. This removes increments of an
1304 /// aligned IV when used by a instruction that ignores the low bits.
1305 bool IndVarSimplify::FoldIVUser(Instruction *UseInst, Instruction *IVOperand) {
1306 Value *IVSrc = 0;
1307 unsigned OperIdx = 0;
1308 const SCEV *FoldedExpr = 0;
1309 switch (UseInst->getOpcode()) {
1310 default:
1311 return false;
1312 case Instruction::UDiv:
1313 case Instruction::LShr:
1314 // We're only interested in the case where we know something about
1315 // the numerator and have a constant denominator.
1316 if (IVOperand != UseInst->getOperand(OperIdx) ||
1317 !isa(UseInst->getOperand(1)))
1318 return false;
1319
1320 // Attempt to fold a binary operator with constant operand.
1321 // e.g. ((I + 1) >> 2) => I >> 2
1322 if (IVOperand->getNumOperands() != 2 ||
1323 !isa(IVOperand->getOperand(1)))
1324 return false;
1325
1326 IVSrc = IVOperand->getOperand(0);
1327 // IVSrc must be the (SCEVable) IV, since the other operand is const.
1328 assert(SE->isSCEVable(IVSrc->getType()) && "Expect SCEVable IV operand");
1329
1330 ConstantInt *D = cast(UseInst->getOperand(1));
1331 if (UseInst->getOpcode() == Instruction::LShr) {
1332 // Get a constant for the divisor. See createSCEV.
1333 uint32_t BitWidth = cast(UseInst->getType())->getBitWidth();
1334 if (D->getValue().uge(BitWidth))
1335 return false;
1336
1337 D = ConstantInt::get(UseInst->getContext(),
1338 APInt(BitWidth, 1).shl(D->getZExtValue()));
1339 }
1340 FoldedExpr = SE->getUDivExpr(SE->getSCEV(IVSrc), SE->getSCEV(D));
1341 }
1342 // We have something that might fold it's operand. Compare SCEVs.
1343 if (!SE->isSCEVable(UseInst->getType()))
1344 return false;
1345
1346 // Bypass the operand if SCEV can prove it has no effect.
1347 if (SE->getSCEV(UseInst) != FoldedExpr)
1348 return false;
1349
1350 DEBUG(dbgs() << "INDVARS: Eliminated IV operand: " << *IVOperand
1351 << " -> " << *UseInst << '\n');
1352
1353 UseInst->setOperand(OperIdx, IVSrc);
1354 assert(SE->getSCEV(UseInst) == FoldedExpr && "bad SCEV with folded oper");
1355
1356 ++NumElimOperand;
1357 Changed = true;
1358 if (IVOperand->use_empty())
1359 DeadInsts.push_back(IVOperand);
1360 return true;
1361 }
1362
13001363 /// pushIVUsers - Add all uses of Def to the current IV's worklist.
13011364 ///
13021365 static void pushIVUsers(
13921455 SimpleIVUsers.pop_back_val();
13931456 // Bypass back edges to avoid extra work.
13941457 if (UseOper.first == CurrIV) continue;
1458
1459 FoldIVUser(UseOper.first, UseOper.second);
13951460
13961461 if (EliminateIVUser(UseOper.first, UseOper.second)) {
13971462 pushIVUsers(UseOper.second, Simplified, SimpleIVUsers);
0 ; RUN: opt < %s -indvars -disable-iv-rewrite -S | FileCheck %s
1
2 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n:32:64"
3
4 ; Indvars should be able to fold IV increments into shr when low bits are zero.
5 ;
6 ; CHECK: @foldIncShr
7 ; CHECK: shr.1 = lshr i32 %0, 5
8 define i32 @foldIncShr(i32* %bitmap, i32 %bit_addr, i32 %nbits) nounwind {
9 entry:
10 br label %while.body
11
12 while.body:
13 %0 = phi i32 [ 0, %entry ], [ %inc.2, %while.body ]
14 %shr = lshr i32 %0, 5
15 %arrayidx = getelementptr inbounds i32* %bitmap, i32 %shr
16 %tmp6 = load i32* %arrayidx, align 4
17 %inc.1 = add i32 %0, 1
18 %shr.1 = lshr i32 %inc.1, 5
19 %arrayidx.1 = getelementptr inbounds i32* %bitmap, i32 %shr.1
20 %tmp6.1 = load i32* %arrayidx.1, align 4
21 %inc.2 = add i32 %inc.1, 1
22 %exitcond.3 = icmp eq i32 %inc.2, 128
23 br i1 %exitcond.3, label %while.end, label %while.body
24
25 while.end:
26 %r = add i32 %tmp6, %tmp6.1
27 ret i32 %r
28 }
29
30 ; Invdars should not fold an increment into shr unless 2^shiftBits is
31 ; a multiple of the recurrence step.
32 ;
33 ; CHECK: @noFoldIncShr
34 ; CHECK: shr.1 = lshr i32 %inc.1, 5
35 define i32 @noFoldIncShr(i32* %bitmap, i32 %bit_addr, i32 %nbits) nounwind {
36 entry:
37 br label %while.body
38
39 while.body:
40 %0 = phi i32 [ 0, %entry ], [ %inc.3, %while.body ]
41 %shr = lshr i32 %0, 5
42 %arrayidx = getelementptr inbounds i32* %bitmap, i32 %shr
43 %tmp6 = load i32* %arrayidx, align 4
44 %inc.1 = add i32 %0, 1
45 %shr.1 = lshr i32 %inc.1, 5
46 %arrayidx.1 = getelementptr inbounds i32* %bitmap, i32 %shr.1
47 %tmp6.1 = load i32* %arrayidx.1, align 4
48 %inc.3 = add i32 %inc.1, 2
49 %exitcond.3 = icmp eq i32 %inc.3, 96
50 br i1 %exitcond.3, label %while.end, label %while.body
51
52 while.end:
53 %r = add i32 %tmp6, %tmp6.1
54 ret i32 %r
55 }