llvm.org GIT mirror llvm / 087bd1e
Make SCEVExpander and LSR more aggressive about hoisting expressions out of loops. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@97642 91177308-0d34-0410-b5e6-96231b3b80d8 Dan Gohman 10 years ago
2 changed file(s) with 258 addition(s) and 61 deletion(s). Raw diff Collapse all Expand all
143143 }
144144 }
145145
146 // Save the original insertion point so we can restore it when we're done.
147 BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
148 BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
149
150 // Move the insertion point out of as many loops as we can.
151 while (const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
152 if (!L->isLoopInvariant(LHS) || !L->isLoopInvariant(RHS)) break;
153 BasicBlock *Preheader = L->getLoopPreheader();
154 if (!Preheader) break;
155
156 // Ok, move up a level.
157 Builder.SetInsertPoint(Preheader, Preheader->getTerminator());
158 }
159
146160 // If we haven't found this binop, insert it.
147161 Value *BO = Builder.CreateBinOp(Opcode, LHS, RHS, "tmp");
148162 rememberInstruction(BO);
163
164 // Restore the original insert point.
165 if (SaveInsertBB)
166 restoreInsertPoint(SaveInsertBB, SaveInsertPt);
167
149168 return BO;
150169 }
151170
492511 }
493512 }
494513
514 // Save the original insertion point so we can restore it when we're done.
515 BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
516 BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
517
518 // Move the insertion point out of as many loops as we can.
519 while (const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
520 if (!L->isLoopInvariant(V) || !L->isLoopInvariant(Idx)) break;
521 BasicBlock *Preheader = L->getLoopPreheader();
522 if (!Preheader) break;
523
524 // Ok, move up a level.
525 Builder.SetInsertPoint(Preheader, Preheader->getTerminator());
526 }
527
495528 // Emit a GEP.
496529 Value *GEP = Builder.CreateGEP(V, Idx, "uglygep");
497530 rememberInstruction(GEP);
531
532 // Restore the original insert point.
533 if (SaveInsertBB)
534 restoreInsertPoint(SaveInsertBB, SaveInsertPt);
535
498536 return GEP;
537 }
538
539 // Save the original insertion point so we can restore it when we're done.
540 BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
541 BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
542
543 // Move the insertion point out of as many loops as we can.
544 while (const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
545 if (!L->isLoopInvariant(V)) break;
546
547 bool AnyIndexNotLoopInvariant = false;
548 for (SmallVectorImpl::const_iterator I = GepIndices.begin(),
549 E = GepIndices.end(); I != E; ++I)
550 if (!L->isLoopInvariant(*I)) {
551 AnyIndexNotLoopInvariant = true;
552 break;
553 }
554 if (AnyIndexNotLoopInvariant)
555 break;
556
557 BasicBlock *Preheader = L->getLoopPreheader();
558 if (!Preheader) break;
559
560 // Ok, move up a level.
561 Builder.SetInsertPoint(Preheader, Preheader->getTerminator());
499562 }
500563
501564 // Insert a pretty getelementptr. Note that this GEP is not marked inbounds,
510573 "scevgep");
511574 Ops.push_back(SE.getUnknown(GEP));
512575 rememberInstruction(GEP);
576
577 // Restore the original insert point.
578 if (SaveInsertBB)
579 restoreInsertPoint(SaveInsertBB, SaveInsertPt);
580
513581 return expand(SE.getAddExpr(Ops));
514582 }
515583
527595 return SC->getValue()->getValue().isNegative();
528596 }
529597
598 /// PickMostRelevantLoop - Given two loops pick the one that's most relevant for
599 /// SCEV expansion. If they are nested, this is the most nested. If they are
600 /// neighboring, pick the later.
601 static const Loop *PickMostRelevantLoop(const Loop *A, const Loop *B,
602 DominatorTree &DT) {
603 if (!A) return B;
604 if (!B) return A;
605 if (A->contains(B)) return B;
606 if (B->contains(A)) return A;
607 if (DT.dominates(A->getHeader(), B->getHeader())) return B;
608 if (DT.dominates(B->getHeader(), A->getHeader())) return A;
609 return A; // Arbitrarily break the tie.
610 }
611
612 /// GetRelevantLoop - Get the most relevant loop associated with the given
613 /// expression, according to PickMostRelevantLoop.
614 static const Loop *GetRelevantLoop(const SCEV *S, LoopInfo &LI,
615 DominatorTree &DT) {
616 if (isa(S))
617 return 0;
618 if (const SCEVUnknown *U = dyn_cast(S)) {
619 if (const Instruction *I = dyn_cast(U->getValue()))
620 return LI.getLoopFor(I->getParent());
621 return 0;
622 }
623 if (const SCEVNAryExpr *N = dyn_cast(S)) {
624 const Loop *L = 0;
625 if (const SCEVAddRecExpr *AR = dyn_cast(S))
626 L = AR->getLoop();
627 for (SCEVNAryExpr::op_iterator I = N->op_begin(), E = N->op_end();
628 I != E; ++I)
629 L = PickMostRelevantLoop(L, GetRelevantLoop(*I, LI, DT), DT);
630 return L;
631 }
632 if (const SCEVCastExpr *C = dyn_cast(S))
633 return GetRelevantLoop(C->getOperand(), LI, DT);
634 if (const SCEVUDivExpr *D = dyn_cast(S))
635 return PickMostRelevantLoop(GetRelevantLoop(D->getLHS(), LI, DT),
636 GetRelevantLoop(D->getRHS(), LI, DT),
637 DT);
638 llvm_unreachable("Unexpected SCEV type!");
639 }
640
641 /// LoopCompare - Compare loops by PickMostRelevantLoop.
642 class LoopCompare {
643 DominatorTree &DT;
644 public:
645 explicit LoopCompare(DominatorTree &dt) : DT(dt) {}
646
647 bool operator()(std::pair LHS,
648 std::pair RHS) const {
649 // Compare loops with PickMostRelevantLoop.
650 if (LHS.first != RHS.first)
651 return PickMostRelevantLoop(LHS.first, RHS.first, DT) != LHS.first;
652
653 // If one operand is a non-constant negative and the other is not,
654 // put the non-constant negative on the right so that a sub can
655 // be used instead of a negate and add.
656 if (isNonConstantNegative(LHS.second)) {
657 if (!isNonConstantNegative(RHS.second))
658 return false;
659 } else if (isNonConstantNegative(RHS.second))
660 return true;
661
662 // Otherwise they are equivalent according to this comparison.
663 return false;
664 }
665 };
666
530667 Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
531 int NumOperands = S->getNumOperands();
532668 const Type *Ty = SE.getEffectiveSCEVType(S->getType());
533669
534 // Find the index of an operand to start with. Choose the operand with
535 // pointer type, if there is one, or the last operand otherwise.
536 int PIdx = 0;
537 for (; PIdx != NumOperands - 1; ++PIdx)
538 if (S->getOperand(PIdx)->getType()->isPointerTy()) break;
539
540 // Expand code for the operand that we chose.
541 Value *V = expand(S->getOperand(PIdx));
542
543 // Turn things like ptrtoint+arithmetic+inttoptr into GEP. See the
544 // comments on expandAddToGEP for details.
545 if (const PointerType *PTy = dyn_cast(V->getType())) {
546 // Take the operand at PIdx out of the list.
547 const SmallVectorImpl &Ops = S->getOperands();
548 SmallVector NewOps;
549 NewOps.insert(NewOps.end(), Ops.begin(), Ops.begin() + PIdx);
550 NewOps.insert(NewOps.end(), Ops.begin() + PIdx + 1, Ops.end());
551 // Make a GEP.
552 return expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, V);
553 }
554
555 // Otherwise, we'll expand the rest of the SCEVAddExpr as plain integer
556 // arithmetic.
557 V = InsertNoopCastOfTo(V, Ty);
558
559 // Emit a bunch of add instructions
560 for (int i = NumOperands-1; i >= 0; --i) {
561 if (i == PIdx) continue;
562 const SCEV *Op = S->getOperand(i);
563 if (isNonConstantNegative(Op)) {
670 // Collect all the add operands in a loop, along with their associated loops.
671 // Iterate in reverse so that constants are emitted last, all else equal, and
672 // so that pointer operands are inserted first, which the code below relies on
673 // to form more involved GEPs.
674 SmallVector, 8> OpsAndLoops;
675 for (std::reverse_iterator I(S->op_end()),
676 E(S->op_begin()); I != E; ++I)
677 OpsAndLoops.push_back(std::make_pair(GetRelevantLoop(*I, *SE.LI, *SE.DT),
678 *I));
679
680 // Sort by loop. Use a stable sort so that constants follow non-constants and
681 // pointer operands precede non-pointer operands.
682 std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(*SE.DT));
683
684 // Emit instructions to add all the operands. Hoist as much as possible
685 // out of loops, and form meaningful getelementptrs where possible.
686 Value *Sum = 0;
687 for (SmallVectorImpl >::iterator
688 I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E; ) {
689 const Loop *CurLoop = I->first;
690 const SCEV *Op = I->second;
691 if (!Sum) {
692 // This is the first operand. Just expand it.
693 Sum = expand(Op);
694 ++I;
695 } else if (const PointerType *PTy = dyn_cast(Sum->getType())) {
696 // The running sum expression is a pointer. Try to form a getelementptr
697 // at this level with that as the base.
698 SmallVector NewOps;
699 for (; I != E && I->first == CurLoop; ++I)
700 NewOps.push_back(I->second);
701 Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, Sum);
702 } else if (const PointerType *PTy = dyn_cast(Op->getType())) {
703 // The running sum is an integer, and there's a pointer at this level.
704 // Try to form a getelementptr.
705 SmallVector NewOps;
706 NewOps.push_back(SE.getUnknown(Sum));
707 for (++I; I != E && I->first == CurLoop; ++I)
708 NewOps.push_back(I->second);
709 Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
710 } else if (isNonConstantNegative(Op)) {
711 // Instead of doing a negate and add, just do a subtract.
564712 Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
565 V = InsertBinop(Instruction::Sub, V, W);
713 Sum = InsertNoopCastOfTo(Sum, Ty);
714 Sum = InsertBinop(Instruction::Sub, Sum, W);
715 ++I;
566716 } else {
717 // A simple add.
567718 Value *W = expandCodeFor(Op, Ty);
568 V = InsertBinop(Instruction::Add, V, W);
569 }
570 }
571 return V;
719 Sum = InsertNoopCastOfTo(Sum, Ty);
720 // Canonicalize a constant to the RHS.
721 if (isa(Sum)) std::swap(Sum, W);
722 Sum = InsertBinop(Instruction::Add, Sum, W);
723 ++I;
724 }
725 }
726
727 return Sum;
572728 }
573729
574730 Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
575731 const Type *Ty = SE.getEffectiveSCEVType(S->getType());
576 int FirstOp = 0; // Set if we should emit a subtract.
577 if (const SCEVConstant *SC = dyn_cast(S->getOperand(0)))
578 if (SC->getValue()->isAllOnesValue())
579 FirstOp = 1;
580
581 int i = S->getNumOperands()-2;
582 Value *V = expandCodeFor(S->getOperand(i+1), Ty);
583
584 // Emit a bunch of multiply instructions
585 for (; i >= FirstOp; --i) {
586 Value *W = expandCodeFor(S->getOperand(i), Ty);
587 V = InsertBinop(Instruction::Mul, V, W);
588 }
589
590 // -1 * ... ---> 0 - ...
591 if (FirstOp == 1)
592 V = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), V);
593 return V;
732
733 // Collect all the mul operands in a loop, along with their associated loops.
734 // Iterate in reverse so that constants are emitted last, all else equal.
735 SmallVector, 8> OpsAndLoops;
736 for (std::reverse_iterator I(S->op_end()),
737 E(S->op_begin()); I != E; ++I)
738 OpsAndLoops.push_back(std::make_pair(GetRelevantLoop(*I, *SE.LI, *SE.DT),
739 *I));
740
741 // Sort by loop. Use a stable sort so that constants follow non-constants.
742 std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(*SE.DT));
743
744 // Emit instructions to mul all the operands. Hoist as much as possible
745 // out of loops.
746 Value *Prod = 0;
747 for (SmallVectorImpl >::iterator
748 I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E; ) {
749 const SCEV *Op = I->second;
750 if (!Prod) {
751 // This is the first operand. Just expand it.
752 Prod = expand(Op);
753 ++I;
754 } else if (Op->isAllOnesValue()) {
755 // Instead of doing a multiply by negative one, just do a negate.
756 Prod = InsertNoopCastOfTo(Prod, Ty);
757 Prod = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), Prod);
758 ++I;
759 } else {
760 // A simple mul.
761 Value *W = expandCodeFor(Op, Ty);
762 Prod = InsertNoopCastOfTo(Prod, Ty);
763 // Canonicalize a constant to the RHS.
764 if (isa(Prod)) std::swap(Prod, W);
765 Prod = InsertBinop(Instruction::Mul, Prod, W);
766 ++I;
767 }
768 }
769
770 return Prod;
594771 }
595772
596773 Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
28732873 Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP)));
28742874 }
28752875
2876 // Flush the operand list to suppress SCEVExpander hoisting.
2877 if (!Ops.empty()) {
2878 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
2879 Ops.clear();
2880 Ops.push_back(SE.getUnknown(FullV));
2881 }
2882
28762883 // Expand the ScaledReg portion.
28772884 Value *ICmpScaledV = 0;
28782885 if (F.AM.Scale != 0) {
28992906 SE.getIntegerSCEV(F.AM.Scale,
29002907 ScaledS->getType()));
29012908 Ops.push_back(ScaledS);
2902 }
2903 }
2904
2905 // Expand the immediate portions.
2906 if (F.AM.BaseGV)
2907 Ops.push_back(SE.getSCEV(F.AM.BaseGV));
2909
2910 // Flush the operand list to suppress SCEVExpander hoisting.
2911 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
2912 Ops.clear();
2913 Ops.push_back(SE.getUnknown(FullV));
2914 }
2915 }
2916
2917 // Expand the GV portion.
2918 if (F.AM.BaseGV) {
2919 Ops.push_back(SE.getUnknown(F.AM.BaseGV));
2920
2921 // Flush the operand list to suppress SCEVExpander hoisting.
2922 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
2923 Ops.clear();
2924 Ops.push_back(SE.getUnknown(FullV));
2925 }
2926
2927 // Expand the immediate portion.
29082928 int64_t Offset = (uint64_t)F.AM.BaseOffs + LF.Offset;
29092929 if (Offset != 0) {
29102930 if (LU.Kind == LSRUse::ICmpZero) {
29192939 } else {
29202940 // Just add the immediate values. These again are expected to be matched
29212941 // as part of the address.
2922 Ops.push_back(SE.getIntegerSCEV(Offset, IntTy));
2942 Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset)));
29232943 }
29242944 }
29252945