llvm.org GIT mirror
When expanding an expression such as (A + B + C + D), sort the operands by loop depth and emit loop-invariant subexpressions outside of loops. This speeds up MultiSource/Applications/viterbi and others. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@97580 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 10 years ago
2 changed file(s) with 129 addition(s) and 39 deletion(s).
 527 527 return SC->getValue()->getValue().isNegative(); 528 528 } 529 529 530 /// PickMostRelevantLoop - Given two loops pick the one that's most relevant for 531 /// SCEV expansion. If they are nested, this is the most nested. If they are 532 /// neighboring, pick the later. 533 static const Loop *PickMostRelevantLoop(const Loop *A, const Loop *B, 534 DominatorTree &DT) { 535 if (!A) return B; 536 if (!B) return A; 537 if (A->contains(B)) return B; 538 if (B->contains(A)) return A; 539 if (DT.dominates(A->getHeader(), B->getHeader())) return B; 540 if (DT.dominates(B->getHeader(), A->getHeader())) return A; 541 return A; // Arbitrarily break the tie. 542 } 543 544 /// GetRelevantLoop - Get the most relevant loop associated with the given 545 /// expression, according to PickMostRelevantLoop. 546 static const Loop *GetRelevantLoop(const SCEV *S, LoopInfo &LI, 547 DominatorTree &DT) { 548 if (isa(S)) 549 return 0; 550 if (const SCEVUnknown *U = dyn_cast(S)) { 551 if (const Instruction *I = dyn_cast(U->getValue())) 552 return LI.getLoopFor(I->getParent()); 553 return 0; 554 } 555 if (const SCEVNAryExpr *N = dyn_cast(S)) { 556 const Loop *L = 0; 557 if (const SCEVAddRecExpr *AR = dyn_cast(S)) 558 L = AR->getLoop(); 559 for (SCEVNAryExpr::op_iterator I = N->op_begin(), E = N->op_end(); 560 I != E; ++I) 561 L = PickMostRelevantLoop(L, GetRelevantLoop(*I, LI, DT), DT); 562 return L; 563 } 564 if (const SCEVCastExpr *C = dyn_cast(S)) 565 return GetRelevantLoop(C->getOperand(), LI, DT); 566 if (const SCEVUDivExpr *D = dyn_cast(S)) 567 return PickMostRelevantLoop(GetRelevantLoop(D->getLHS(), LI, DT), 568 GetRelevantLoop(D->getRHS(), LI, DT), 569 DT); 570 llvm_unreachable("Unexpected SCEV type!"); 571 } 572 573 namespace { 574 575 /// LoopCompare - Compare loops by PickMostRelevantLoop. 576 class LoopCompare { 577 DominatorTree &DT; 578 public: 579 explicit LoopCompare(DominatorTree &dt) : DT(dt) {} 580 581 bool operator()(std::pair LHS, 582 std::pair RHS) const { 583 // Compare loops with PickMostRelevantLoop. 584 if (LHS.first != RHS.first) 585 return PickMostRelevantLoop(LHS.first, RHS.first, DT) == LHS.first; 586 587 // If one operand is a non-constant negative and the other is not, 588 // put the non-constant negative on the right so that a sub can 589 // be used instead of a negate and add. 590 if (isNonConstantNegative(LHS.second)) { 591 if (!isNonConstantNegative(RHS.second)) 592 return false; 593 } else if (isNonConstantNegative(RHS.second)) 594 return true; 595 596 // Otherwise they are equivalent according to this comparison. 597 return false; 598 } 599 }; 600 601 } 602 530 603 Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { 531 int NumOperands = S->getNumOperands(); 532 604 const Type *Ty = SE.getEffectiveSCEVType(S->getType()); 533 605 534 // Find the index of an operand to start with. Choose the operand with 535 // pointer type, if there is one, or the last operand otherwise. 536 int PIdx = 0; 537 for (; PIdx != NumOperands - 1; ++PIdx) 538 if (S->getOperand(PIdx)->getType()->isPointerTy()) break; 539 540 // Expand code for the operand that we chose. 541 Value *V = expand(S->getOperand(PIdx)); 542 543 // Turn things like ptrtoint+arithmetic+inttoptr into GEP. See the 544 // comments on expandAddToGEP for details. 545 if (const PointerType *PTy = dyn_cast(V->getType())) { 546 // Take the operand at PIdx out of the list. 547 const SmallVectorImpl &Ops = S->getOperands(); 548 SmallVector NewOps; 549 NewOps.insert(NewOps.end(), Ops.begin(), Ops.begin() + PIdx); 550 NewOps.insert(NewOps.end(), Ops.begin() + PIdx + 1, Ops.end()); 551 // Make a GEP. 552 return expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, V); 553 } 554 555 // Otherwise, we'll expand the rest of the SCEVAddExpr as plain integer 556 // arithmetic. 557 V = InsertNoopCastOfTo(V, Ty); 558 559 // Emit a bunch of add instructions 560 for (int i = NumOperands-1; i >= 0; --i) { 561 if (i == PIdx) continue; 562 const SCEV *Op = S->getOperand(i); 563 if (isNonConstantNegative(Op)) {⏎ 606 // Collect all the add operands in a loop, along with their associated loops.⏎ 607 // Iterate in reverse so that constants are emitted last, all else equal, and 608 // so that pointer operands are inserted first, which the code below relies on 609 // to form more involved GEPs. 610 SmallVector, 8> OpsAndLoops; 611 for (std::reverse_iterator I(S->op_end()), 612 E(S->op_begin()); I != E; ++I) 613 OpsAndLoops.push_back(std::make_pair(GetRelevantLoop(*I, *SE.LI, *SE.DT), 614 *I)); 615 616 // Sort by loop. Use a stable sort so that constants follow non-constants and 617 // pointer operands precede non-pointer operands. 618 std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(*SE.DT)); 619 620 // Emit instructions to add all the operands. Hoist as much as possible 621 // out of loops, and form meaningful getelementptrs where possible. 622 Value *Sum = 0; 623 for (SmallVectorImpl >::iterator 624 I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E; ) { 625 const Loop *CurLoop = I->first; 626 const SCEV *Op = I->second; 627 if (!Sum) { 628 // This is the first operand. Just expand it. 629 Sum = expand(Op); 630 ++I; 631 } else if (const PointerType *PTy = dyn_cast(Sum->getType())) { 632 // The running sum expression is a pointer. Try to form a getelementptr 633 // at this level with that as the base. 634 SmallVector NewOps; 635 for (; I != E && I->first == CurLoop; ++I) 636 NewOps.push_back(I->second); 637 Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, Sum); 638 } else if (const PointerType *PTy = dyn_cast(Op->getType())) { 639 // The running sum is an integer, and there's a pointer at this level. 640 // Try to form a getelementptr. 641 SmallVector NewOps; 642 NewOps.push_back(SE.getUnknown(Sum)); 643 for (++I; I != E && I->first == CurLoop; ++I) 644 NewOps.push_back(I->second); 645 Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op)); 646 } else if (isNonConstantNegative(Op)) { 647 // Instead of doing a negate and add, just do a subtract. 564 648 Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty); 565 V = InsertBinop(Instruction::Sub, V, W);⏎ 649 Sum = InsertNoopCastOfTo(Sum, Ty);⏎ 650 Sum = InsertBinop(Instruction::Sub, Sum, W); 651 ++I; 566 652 } else { 653 // A simple add. 567 654 Value *W = expandCodeFor(Op, Ty); 568 V = InsertBinop(Instruction::Add, V, W); 569 } 570 } 571 return V;⏎ 655 Sum = InsertNoopCastOfTo(Sum, Ty);⏎ 656 Sum = InsertBinop(Instruction::Add, Sum, W); 657 ++I; 658 } 659 } 660 661 return Sum; 572 662 } 573 663 574 664 Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
 None ; RUN: llc < %s -march=x86 -stats |& grep {Number of loads added} | grep 2 1 ; RUN: llc < %s -march=x86 -stats |& grep {Number of register spills} | grep 1 2 ; RUN: llc < %s -march=x86 -stats |& grep {Number of machine instrs printed} | grep 34⏎ 0 ; RUN: llc < %s -march=x86 -stats |& not grep {Number of loads added}⏎ 1 ; RUN: llc < %s -march=x86 -stats |& not grep {Number of register spills} 2 ; RUN: llc < %s -march=x86 -stats |& grep {Number of machine instrs printed} | grep 32 3 3 ; PR3495 4 4 5 5 target triple = "i386-pc-linux-gnu"