llvm.org GIT mirror llvm / fe647d2
Make processing @llvm.assume more efficient by using operand bundles There was an efficiency problem with how we processed @llvm.assume in ValueTracking (and other places). The AssumptionCache tracked all of the assumptions in a given function. In order to find assumptions relevant to computing known bits, etc. we searched every assumption in the function. For ValueTracking, that means that we did O(#assumes * #values) work in InstCombine and other passes (with a constant factor that can be quite large because we'd repeat this search at every level of recursion of the analysis). Several of us discussed this situation at the last developers' meeting, and this implements the discussed solution: Make the values that an assume might affect operands of the assume itself. To avoid exposing this detail to frontends and passes that need not worry about it, I've used the new operand-bundle feature to add these extra call "operands" in a way that does not affect the intrinsic's signature. I think this solution is relatively clean. InstCombine adds these extra operands based on what ValueTracking, LVI, etc. will need and then those passes need only search the users of the values under consideration. This should fix the computational-complexity problem. At this point, no passes depend on the AssumptionCache, and so I'll remove that as a follow-up change. Differential Revision: https://reviews.llvm.org/D27259 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@289755 91177308-0d34-0410-b5e6-96231b3b80d8 Hal Finkel 2 years ago
19 changed file(s) with 284 addition(s) and 154 deletion(s). Raw diff Collapse all Expand all
17511751 site, these bundles may contain any values that are needed by the
17521752 generated code. For more details, see :ref:`GC Transitions
17531753 `.
1754
1755 Affected Operand Bundles
1756 ^^^^^^^^^^^^^^^^^^^^^^^^
1757
1758 Affected operand bundles are characterized by the ``"affected"`` operand bundle
1759 tag. These operand bundles indicate that a call, specifically a call to an
1760 intrinsic like ``llvm.assume``, implies some additional knowledge about the
1761 values within the bundle. This enables the optimizer to efficiently find these
1762 relationships. The optimizer will add these automatically.
17541763
17551764 .. _moduleasm:
17561765
531531 ///
532532 ValueExprMapType ValueExprMap;
533533
534 /// This is a map of SCEVs to intrinsics (e.g. assumptions) that might affect
535 /// (i.e. imply something about) them.
536 DenseMap> AffectedMap;
537
534538 /// Mark predicate values currently being processed by isImpliedCond.
535539 SmallPtrSet PendingLoopPredicates;
536540
799803 ConstantRange getRangeViaFactoring(const SCEV *Start, const SCEV *Stop,
800804 const SCEV *MaxBECount, unsigned BitWidth);
801805
806 /// Add to the AffectedMap this SCEV if its operands are in the AffectedMap.
807 void addAffectedFromOperands(const SCEV *S);
808
802809 /// We know that there is no SCEV for the specified value. Analyze the
803810 /// expression.
804811 const SCEV *createSCEV(Value *V);
7575 SmallPtrSet Visited;
7676 SmallVector Worklist;
7777
78 for (auto &AssumeVH : AC->assumptions()) {
79 if (!AssumeVH)
80 continue;
81 Instruction *I = cast(AssumeVH);
82
83 // Filter out call sites outside of the loop so we don't do a function's
84 // worth of work for each of its loops (and, in the common case, ephemeral
85 // values in the loop are likely due to @llvm.assume calls in the loop).
86 if (!L->contains(I->getParent()))
87 continue;
88
89 if (EphValues.insert(I).second)
90 appendSpeculatableOperands(I, Visited, Worklist);
91 }
78 for (auto &B : L->blocks())
79 for (auto &I : *B)
80 if (auto *II = dyn_cast(&I))
81 if (II->getIntrinsicID() == Intrinsic::assume &&
82 EphValues.insert(II).second)
83 appendSpeculatableOperands(II, Visited, Worklist);
9284
9385 completeEphemeralValues(Visited, Worklist, EphValues);
9486 }
9991 SmallPtrSet Visited;
10092 SmallVector Worklist;
10193
102 for (auto &AssumeVH : AC->assumptions()) {
103 if (!AssumeVH)
104 continue;
105 Instruction *I = cast(AssumeVH);
106 assert(I->getParent()->getParent() == F &&
107 "Found assumption for the wrong function!");
108
109 if (EphValues.insert(I).second)
110 appendSpeculatableOperands(I, Visited, Worklist);
111 }
94 for (auto &B : *F)
95 for (auto &I : B)
96 if (auto *II = dyn_cast(&I))
97 if (II->getIntrinsicID() == Intrinsic::assume &&
98 EphValues.insert(II).second)
99 appendSpeculatableOperands(II, Visited, Worklist);
112100
113101 completeEphemeralValues(Visited, Worklist, EphValues);
114102 }
923923 if (!BBI)
924924 return;
925925
926 for (auto &AssumeVH : AC->assumptions()) {
927 if (!AssumeVH)
926 for (auto *U : Val->users()) {
927 auto *II = dyn_cast(U);
928 if (!II)
928929 continue;
929 auto *I = cast(AssumeVH);
930 if (!isValidAssumeForContext(I, BBI, DT))
930 if (II->getIntrinsicID() != Intrinsic::assume)
931931 continue;
932
933 BBLV = intersect(BBLV, getValueFromCondition(Val, I->getArgOperand(0)));
932 if (!isValidAssumeForContext(II, BBI, DT))
933 continue;
934
935 BBLV = intersect(BBLV, getValueFromCondition(Val, II->getArgOperand(0)));
934936 }
935937
936938 // If guards are not used in the module, don't spend time looking for them
12111211 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
12121212 Op, Ty);
12131213 UniqueSCEVs.InsertNode(S, IP);
1214 addAffectedFromOperands(S);
12141215 return S;
12151216 }
12161217
15971598 // these to prove lack of overflow. Use this fact to avoid
15981599 // doing extra work that may not pay off.
15991600 if (!isa(MaxBECount) || HasGuards ||
1600 !AC.assumptions().empty()) {
1601 !AffectedMap.empty()) {
16011602 // If the backedge is guarded by a comparison with the pre-inc
16021603 // value the addrec is safe. Also, if the entry is guarded by
16031604 // a comparison with the start value and the backedge is
16631664 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
16641665 Op, Ty);
16651666 UniqueSCEVs.InsertNode(S, IP);
1667 addAffectedFromOperands(S);
16661668 return S;
16671669 }
16681670
18321834 // doing extra work that may not pay off.
18331835
18341836 if (!isa(MaxBECount) || HasGuards ||
1835 !AC.assumptions().empty()) {
1837 !AffectedMap.empty()) {
18361838 // If the backedge is guarded by a comparison with the pre-inc
18371839 // value the addrec is safe. Also, if the entry is guarded by
18381840 // a comparison with the start value and the backedge is
18901892 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
18911893 Op, Ty);
18921894 UniqueSCEVs.InsertNode(S, IP);
1895 addAffectedFromOperands(S);
18931896 return S;
18941897 }
18951898
24432446 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
24442447 O, Ops.size());
24452448 UniqueSCEVs.InsertNode(S, IP);
2449 addAffectedFromOperands(S);
24462450 }
24472451 S->setNoWrapFlags(Flags);
24482452 return S;
27352739 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
27362740 O, Ops.size());
27372741 UniqueSCEVs.InsertNode(S, IP);
2742 addAffectedFromOperands(S);
27382743 }
27392744 S->setNoWrapFlags(Flags);
27402745 return S;
28552860 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
28562861 LHS, RHS);
28572862 UniqueSCEVs.InsertNode(S, IP);
2863 addAffectedFromOperands(S);
28582864 return S;
28592865 }
28602866
30353041 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
30363042 O, Operands.size(), L);
30373043 UniqueSCEVs.InsertNode(S, IP);
3044 addAffectedFromOperands(S);
30383045 }
30393046 S->setNoWrapFlags(Flags);
30403047 return S;
31903197 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
31913198 O, Ops.size());
31923199 UniqueSCEVs.InsertNode(S, IP);
3200 addAffectedFromOperands(S);
31933201 return S;
31943202 }
31953203
32913299 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
32923300 O, Ops.size());
32933301 UniqueSCEVs.InsertNode(S, IP);
3302 addAffectedFromOperands(S);
32943303 return S;
32953304 }
32963305
34913500 ExprValueMap[Stripped].insert({V, Offset});
34923501 }
34933502 }
3503
3504 // If this value is an instruction or an argument, and might be affected by
3505 // an assumption, and its SCEV to the AffectedMap.
3506 if (isa(V) || isa(V)) {
3507 for (auto *U : V->users()) {
3508 auto *II = dyn_cast(U);
3509 if (!II)
3510 continue;
3511 if (II->getIntrinsicID() != Intrinsic::assume)
3512 continue;
3513
3514 AffectedMap[S].insert(II);
3515 }
3516 }
3517
34943518 return S;
3519 }
3520
3521 // If one of this SCEV's operands is in the AffectedMap (meaning that it might
3522 // be affected by an assumption), then this SCEV might be affected by the same
3523 // assumption.
3524 void ScalarEvolution::addAffectedFromOperands(const SCEV *S) {
3525 if (auto *NS = dyn_cast(S))
3526 for (auto *Op : NS->operands()) {
3527 auto AMI = AffectedMap.find(Op);
3528 if (AMI == AffectedMap.end())
3529 continue;
3530
3531 AffectedMap[S].insert(AMI->second.begin(), AMI->second.end());
3532 }
34953533 }
34963534
34973535 const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
79257963 }
79267964
79277965 // Check conditions due to any @llvm.assume intrinsics.
7928 for (auto &AssumeVH : AC.assumptions()) {
7929 if (!AssumeVH)
7930 continue;
7931 auto *CI = cast(AssumeVH);
7932 if (!DT.dominates(CI, Latch->getTerminator()))
7933 continue;
7934
7935 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
7936 return true;
7937 }
7966 auto CheckAssumptions = [&](const SCEV *S) {
7967 auto AMI = AffectedMap.find(S);
7968 if (AMI != AffectedMap.end())
7969 for (auto *Assume : AMI->second) {
7970 auto *CI = cast(Assume);
7971 if (!DT.dominates(CI, Latch->getTerminator()))
7972 continue;
7973
7974 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
7975 return true;
7976 }
7977
7978 return false;
7979 };
7980
7981 if (CheckAssumptions(LHS) || CheckAssumptions(RHS))
7982 return true;
79387983
79397984 // If the loop is not reachable from the entry block, we risk running into an
79407985 // infinite loop as we walk up into the dom tree. These loops do not matter
80198064 }
80208065
80218066 // Check conditions due to any @llvm.assume intrinsics.
8022 for (auto &AssumeVH : AC.assumptions()) {
8023 if (!AssumeVH)
8024 continue;
8025 auto *CI = cast(AssumeVH);
8026 if (!DT.dominates(CI, L->getHeader()))
8027 continue;
8028
8029 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
8030 return true;
8031 }
8067 auto CheckAssumptions = [&](const SCEV *S) {
8068 auto AMI = AffectedMap.find(S);
8069 if (AMI != AffectedMap.end())
8070 for (auto *Assume : AMI->second) {
8071 auto *CI = cast(Assume);
8072 if (!DT.dominates(CI, L->getHeader()))
8073 continue;
8074
8075 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
8076 return true;
8077 }
8078
8079 return false;
8080 };
8081
8082 if (CheckAssumptions(LHS) || CheckAssumptions(RHS))
8083 return true;
80328084
80338085 return false;
80348086 }
525525
526526 unsigned BitWidth = KnownZero.getBitWidth();
527527
528 for (auto &AssumeVH : Q.AC->assumptions()) {
529 if (!AssumeVH)
528 for (auto *U : V->users()) {
529 auto *II = dyn_cast(U);
530 if (!II)
530531 continue;
531 CallInst *I = cast(AssumeVH);
532 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
533 "Got assumption for the wrong function!");
534 if (Q.isExcluded(I))
532 if (II->getIntrinsicID() != Intrinsic::assume)
535533 continue;
536
537 // Warning: This loop can end up being somewhat performance sensetive.
538 // We're running this loop for once for each value queried resulting in a
539 // runtime of ~O(#assumes * #values).
540
541 assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume &&
542 "must be an assume intrinsic");
543
544 Value *Arg = I->getArgOperand(0);
545
546 if (Arg == V && isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
534 if (Q.isExcluded(II))
535 continue;
536
537 Value *Arg = II->getArgOperand(0);
538
539 if (Arg == V && isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
547540 assert(BitWidth == 1 && "assume operand is not i1?");
548541 KnownZero.clearAllBits();
549542 KnownOne.setAllBits();
550543 return;
551544 }
552545
546 // Note that the patterns below need to be kept in sync with the code
547 // in InstCombiner::visitCallInst that adds relevant values to each
548 // assume's operand bundles.
549
553550 // The remaining tests are all recursive, so bail out if we hit the limit.
554551 if (Depth == MaxDepth)
555552 continue;
563560 ConstantInt *C;
564561 // assume(v = a)
565562 if (match(Arg, m_c_ICmp(Pred, m_V, m_Value(A))) &&
566 Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
563 Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
567564 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
568 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
565 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
569566 KnownZero |= RHSKnownZero;
570567 KnownOne |= RHSKnownOne;
571568 // assume(v & b = a)
572569 } else if (match(Arg,
573570 m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) &&
574571 Pred == ICmpInst::ICMP_EQ &&
575 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
572 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
576573 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
577 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
574 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
578575 APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0);
579 computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, I));
576 computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, II));
580577
581578 // For those bits in the mask that are known to be one, we can propagate
582579 // known bits from the RHS to V.
586583 } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))),
587584 m_Value(A))) &&
588585 Pred == ICmpInst::ICMP_EQ &&
589 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
586 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
590587 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
591 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
588 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
592589 APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0);
593 computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, I));
590 computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, II));
594591
595592 // For those bits in the mask that are known to be one, we can propagate
596593 // inverted known bits from the RHS to V.
600597 } else if (match(Arg,
601598 m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) &&
602599 Pred == ICmpInst::ICMP_EQ &&
603 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
600 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
604601 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
605 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
602 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
606603 APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
607 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
604 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
608605
609606 // For those bits in B that are known to be zero, we can propagate known
610607 // bits from the RHS to V.
614611 } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))),
615612 m_Value(A))) &&
616613 Pred == ICmpInst::ICMP_EQ &&
617 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
614 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
618615 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
619 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
616 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
620617 APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
621 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
618 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
622619
623620 // For those bits in B that are known to be zero, we can propagate
624621 // inverted known bits from the RHS to V.
628625 } else if (match(Arg,
629626 m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) &&
630627 Pred == ICmpInst::ICMP_EQ &&
631 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
628 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
632629 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
633 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
630 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
634631 APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
635 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
632 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
636633
637634 // For those bits in B that are known to be zero, we can propagate known
638635 // bits from the RHS to V. For those bits in B that are known to be one,
645642 } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))),
646643 m_Value(A))) &&
647644 Pred == ICmpInst::ICMP_EQ &&
648 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
645 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
649646 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
650 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
647 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
651648 APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
652 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
649 computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
653650
654651 // For those bits in B that are known to be zero, we can propagate
655652 // inverted known bits from the RHS to V. For those bits in B that are
662659 } else if (match(Arg, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)),
663660 m_Value(A))) &&
664661 Pred == ICmpInst::ICMP_EQ &&
665 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
662 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
666663 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
667 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
664 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
668665 // For those bits in RHS that are known, we can propagate them to known
669666 // bits in V shifted to the right by C.
670667 KnownZero |= RHSKnownZero.lshr(C->getZExtValue());
673670 } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))),
674671 m_Value(A))) &&
675672 Pred == ICmpInst::ICMP_EQ &&
676 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
673 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
677674 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
678 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
675 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
679676 // For those bits in RHS that are known, we can propagate them inverted
680677 // to known bits in V shifted to the right by C.
681678 KnownZero |= RHSKnownOne.lshr(C->getZExtValue());
686683 m_AShr(m_V, m_ConstantInt(C))),
687684 m_Value(A))) &&
688685 Pred == ICmpInst::ICMP_EQ &&
689 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
686 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
690687 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
691 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
688 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
692689 // For those bits in RHS that are known, we can propagate them to known
693690 // bits in V shifted to the right by C.
694691 KnownZero |= RHSKnownZero << C->getZExtValue();
699696 m_AShr(m_V, m_ConstantInt(C)))),
700697 m_Value(A))) &&
701698 Pred == ICmpInst::ICMP_EQ &&
702 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
699 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
703700 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
704 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
701 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
705702 // For those bits in RHS that are known, we can propagate them inverted
706703 // to known bits in V shifted to the right by C.
707704 KnownZero |= RHSKnownOne << C->getZExtValue();
709706 // assume(v >=_s c) where c is non-negative
710707 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
711708 Pred == ICmpInst::ICMP_SGE &&
712 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
709 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
713710 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
714 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
711 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
715712
716713 if (RHSKnownZero.isNegative()) {
717714 // We know that the sign bit is zero.
720717 // assume(v >_s c) where c is at least -1.
721718 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
722719 Pred == ICmpInst::ICMP_SGT &&
723 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
720 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
724721 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
725 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
722 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
726723
727724 if (RHSKnownOne.isAllOnesValue() || RHSKnownZero.isNegative()) {
728725 // We know that the sign bit is zero.
731728 // assume(v <=_s c) where c is negative
732729 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
733730 Pred == ICmpInst::ICMP_SLE &&
734 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
731 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
735732 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
736 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
733 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
737734
738735 if (RHSKnownOne.isNegative()) {
739736 // We know that the sign bit is one.
742739 // assume(v <_s c) where c is non-positive
743740 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
744741 Pred == ICmpInst::ICMP_SLT &&
745 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
742 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
746743 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
747 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
744 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
748745
749746 if (RHSKnownZero.isAllOnesValue() || RHSKnownOne.isNegative()) {
750747 // We know that the sign bit is one.
753750 // assume(v <=_u c)
754751 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
755752 Pred == ICmpInst::ICMP_ULE &&
756 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
753 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
757754 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
758 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
755 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
759756
760757 // Whatever high bits in c are zero are known to be zero.
761758 KnownZero |=
763760 // assume(v <_u c)
764761 } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
765762 Pred == ICmpInst::ICMP_ULT &&
766 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
763 isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
767764 APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
768 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
765 computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
769766
770767 // Whatever high bits in c are zero are known to be zero (if c is a power
771768 // of 2, then one more).
772 if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I)))
769 if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, II)))
773770 KnownZero |=
774771 APInt::getHighBitsSet(BitWidth, RHSKnownZero.countLeadingOnes()+1);
775772 else
25172517 if (KnownOne.isAllOnesValue())
25182518 return eraseInstFromFunction(*II);
25192519
2520 // For assumptions, add to the associated operand bundle the values to which
2521 // the assumption might apply.
2522 // Note: This code must be kept in-sync with the code in
2523 // computeKnownBitsFromAssume in ValueTracking.
2524 SmallVector Affected;
2525 auto AddAffected = [&Affected](Value *V) {
2526 if (isa(V)) {
2527 Affected.push_back(V);
2528 } else if (auto *I = dyn_cast(V)) {
2529 Affected.push_back(I);
2530
2531 if (I->getOpcode() == Instruction::BitCast ||
2532 I->getOpcode() == Instruction::PtrToInt) {
2533 V = I->getOperand(0);
2534 if (isa(V) || isa(V))
2535 Affected.push_back(V);
2536 }
2537 }
2538 };
2539
2540 CmpInst::Predicate Pred;
2541 if (match(IIOperand, m_ICmp(Pred, m_Value(A), m_Value(B)))) {
2542 AddAffected(A);
2543 AddAffected(B);
2544
2545 if (Pred == ICmpInst::ICMP_EQ) {
2546 // For equality comparisons, we handle the case of bit inversion.
2547 auto AddAffectedFromEq = [&AddAffected](Value *V) {
2548 Value *A;
2549 if (match(V, m_Not(m_Value(A)))) {
2550 AddAffected(A);
2551 V = A;
2552 }
2553
2554 Value *B;
2555 ConstantInt *C;
2556 if (match(V,
2557 m_CombineOr(m_And(m_Value(A), m_Value(B)),
2558 m_CombineOr(m_Or(m_Value(A), m_Value(B)),
2559 m_Xor(m_Value(A), m_Value(B)))))) {
2560 AddAffected(A);
2561 AddAffected(B);
2562 } else if (match(V,
2563 m_CombineOr(m_Shl(m_Value(A), m_ConstantInt(C)),
2564 m_CombineOr(m_LShr(m_Value(A), m_ConstantInt(C)),
2565 m_AShr(m_Value(A),
2566 m_ConstantInt(C)))))) {
2567 AddAffected(A);
2568 }
2569 };
2570
2571 AddAffectedFromEq(A);
2572 AddAffectedFromEq(B);
2573 }
2574 }
2575
2576 // If the list of affected values is the same as the existing list then
2577 // there's nothing more to do here.
2578 if (!Affected.empty())
2579 if (auto OB = CI.getOperandBundle("affected"))
2580 if (Affected.size() == OB.getValue().Inputs.size() &&
2581 std::equal(Affected.begin(), Affected.end(),
2582 OB.getValue().Inputs.begin()))
2583 Affected.clear();
2584
2585 if (!Affected.empty()) {
2586 Builder->CreateCall(AssumeIntrinsic, IIOperand,
2587 OperandBundleDef("affected", Affected),
2588 II->getName());
2589 return eraseInstFromFunction(*II);
2590 }
2591
25202592 break;
25212593 }
25222594 case Intrinsic::experimental_gc_relocate: {
424424 NewSrcAlignments.clear();
425425
426426 bool Changed = false;
427 for (auto &AssumeVH : AC.assumptions())
428 if (AssumeVH)
429 Changed |= processAssumption(cast(AssumeVH));
427
428 for (auto &B : F)
429 for (auto &I : B)
430 if (auto *II = dyn_cast(&I))
431 if (II->getIntrinsicID() == Intrinsic::assume)
432 Changed |= processAssumption(II);
430433
431434 return Changed;
432435 }
5454 %cmp = icmp slt i32 %iv, 10000
5555 ; CHECK: %iv.sext = sext i32 %iv to i64
5656 ; CHECK-NEXT: --> {0,+,3}<%loop>
57 call void @llvm.assume(i1 %cmp)
57 call void @llvm.assume(i1 %cmp) [ "affected"(i32 %iv) ]
5858 %c = load volatile i1, i1* %cond
5959 br i1 %c, label %loop, label %leave
6060
158158 %cmp = icmp ugt i32 %iv.inc, -10000
159159 ; CHECK: %iv.zext = zext i32 %iv to i64
160160 ; CHECK-NEXT: --> {30000,+,-2}<%loop>
161 call void @llvm.assume(i1 %cmp)
161 call void @llvm.assume(i1 %cmp) [ "affected"(i32 %iv.inc) ]
162162 %c = load volatile i1, i1* %cond
163163 br i1 %c, label %loop, label %leave
164164
1010 entry:
1111 %n = and i32 %no, 4294967294
1212 %0 = icmp sgt i32 %n, 0 ; [#uses=1]
13 tail call void @llvm.assume(i1 %0)
13 tail call void @llvm.assume(i1 %0) [ "affected"(i32 %n) ]
1414 br label %bb.nph
1515
1616 bb.nph: ; preds = %entry
2525 define i8 @test2(i8 %a) {
2626 ; CHECK-LABEL: @test2
2727 %cmp1 = icmp eq i8 %a, 5
28 call void @llvm.assume(i1 %cmp1)
28 call void @llvm.assume(i1 %cmp1) [ "affected"(i8 %a) ]
2929 %cmp2 = icmp eq i8 %a, 3
3030 ; CHECK: br i1 false, label %dead, label %exit
3131 br i1 %cmp2, label %dead, label %exit
4242 dead:
4343 %cmp2 = icmp eq i8 %a, 3
4444 ; CHECK: call void @llvm.assume(i1 false)
45 call void @llvm.assume(i1 %cmp2)
45 call void @llvm.assume(i1 %cmp2) [ "affected"(i8 %a) ]
4646 ret i8 %a
4747 exit:
4848 ret i8 0
1010 define void @_Z3fooR1s(%struct.s* nocapture readonly dereferenceable(8) %x) #0 {
1111
1212 ; CHECK-LABEL: @_Z3fooR1s
13 ; CHECK: call void @llvm.assume
13 ; CHECK: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, double* %{{.*}}) ]
1414 ; CHECK-NOT: call void @llvm.assume
1515
1616 entry:
1010 ; been removed:
1111 ; CHECK-LABEL: @foo1
1212 ; CHECK-DAG: load i32, i32* %a, align 32
13 ; CHECK-DAG: call void @llvm.assume
13 ; CHECK-DAG: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, i32* %a) ]
1414 ; CHECK: ret i32
1515
1616 %ptrint = ptrtoint i32* %a to i64
2727 ; Same check as in @foo1, but make sure it works if the assume is first too.
2828 ; CHECK-LABEL: @foo2
2929 ; CHECK-DAG: load i32, i32* %a, align 32
30 ; CHECK-DAG: call void @llvm.assume
30 ; CHECK-DAG: call void @llvm.assume(i1 %maskcond) [ "affected"(i64 %maskedptr, i64 %ptrint, i32* %a) ]
3131 ; CHECK: ret i32
3232
3333 %ptrint = ptrtoint i32* %a to i64
5050 ; CHECK: ret i32 4
5151
5252 %cmp = icmp eq i32 %a, 4
53 tail call void @llvm.assume(i1 %cmp)
53 tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
5454 ret i32 %a
5555 }
5656
9292 %and1 = and i32 %a, 3
9393
9494 ; CHECK-LABEL: @bar1
95 ; CHECK: call void @llvm.assume
95 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ]
9696 ; CHECK: ret i32 1
9797
9898 %and = and i32 %a, 7
106106 define i32 @bar2(i32 %a) #0 {
107107 entry:
108108 ; CHECK-LABEL: @bar2
109 ; CHECK: call void @llvm.assume
109 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ]
110110 ; CHECK: ret i32 1
111111
112112 %and = and i32 %a, 7
124124
125125 ; Don't be fooled by other assumes around.
126126 ; CHECK-LABEL: @bar3
127 ; CHECK: call void @llvm.assume
127 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ]
128128 ; CHECK: ret i32 1
129129
130130 tail call void @llvm.assume(i1 %x)
144144 %and1 = and i32 %b, 3
145145
146146 ; CHECK-LABEL: @bar4
147 ; CHECK: call void @llvm.assume
148 ; CHECK: call void @llvm.assume
147 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ]
148 ; CHECK: call void @llvm.assume(i1 %cmp2) [ "affected"(i32 %a, i32 %b) ]
149149 ; CHECK: ret i32 1
150150
151151 %and = and i32 %a, 7
166166 ret i32 %conv
167167
168168 ; CHECK-LABEL: @icmp1
169 ; CHECK: call void @llvm.assume
169 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
170170 ; CHECK: ret i32 1
171171
172172 }
181181 ret i32 %lnot.ext
182182
183183 ; CHECK-LABEL: @icmp2
184 ; CHECK: call void @llvm.assume
184 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
185185 ; CHECK: ret i32 0
186186 }
187187
216216
217217 ; CHECK-LABEL: @nonnull2
218218 ; CHECK-NOT: !nonnull
219 ; CHECK: call void @llvm.assume
219 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %load) ]
220220 }
221221
222222 ; Make sure the above canonicalization does not trigger
235235
236236 ; CHECK-LABEL: @nonnull3
237237 ; CHECK-NOT: !nonnull
238 ; CHECK: call void @llvm.assume
238 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32* %load) ]
239239 }
240240
241241 ; Make sure the above canonicalization does not trigger
253253
254254 ; CHECK-LABEL: @nonnull4
255255 ; CHECK-NOT: !nonnull
256 ; CHECK: call void @llvm.assume
256 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32* %load) ]
257257 }
258258
259259
88 define i32 @test1(i32 %a) #0 {
99 entry:
1010 ; CHECK-LABEL: @test1
11 ; CHECK: call void @llvm.assume
11 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %and, i32 %a) ]
1212 ; CHECK: ret i32 5
1313
1414 %and = and i32 %a, 15
2323 define i32 @test2(i32 %a) #0 {
2424 entry:
2525 ; CHECK-LABEL: @test2
26 ; CHECK: call void @llvm.assume
26 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a.not, i32 %a) ]
2727 ; CHECK: ret i32 2
2828
2929 %and = and i32 %a, 15
3939 define i32 @test3(i32 %a) #0 {
4040 entry:
4141 ; CHECK-LABEL: @test3
42 ; CHECK: call void @llvm.assume
42 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v, i32 %a) ]
4343 ; CHECK: ret i32 5
4444
4545 %v = or i32 %a, 4294967280
5454 define i32 @test4(i32 %a) #0 {
5555 entry:
5656 ; CHECK-LABEL: @test4
57 ; CHECK: call void @llvm.assume
57 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a.not, i32 %a) ]
5858 ; CHECK: ret i32 2
5959
6060 %v = or i32 %a, 4294967280
7070 define i32 @test5(i32 %a) #0 {
7171 entry:
7272 ; CHECK-LABEL: @test5
73 ; CHECK: call void @llvm.assume
73 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
7474 ; CHECK: ret i32 4
7575
7676 %v = xor i32 %a, 1
8585 define i32 @test6(i32 %a) #0 {
8686 entry:
8787 ; CHECK-LABEL: @test6
88 ; CHECK: call void @llvm.assume
88 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ]
8989 ; CHECK: ret i32 5
9090
9191 %v = shl i32 %a, 2
100100 define i32 @test7(i32 %a) #0 {
101101 entry:
102102 ; CHECK-LABEL: @test7
103 ; CHECK: call void @llvm.assume
103 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ]
104104 ; CHECK: ret i32 20
105105
106106 %v = lshr i32 %a, 2
115115 define i32 @test8(i32 %a) #0 {
116116 entry:
117117 ; CHECK-LABEL: @test8
118 ; CHECK: call void @llvm.assume
118 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %v.mask, i32 %a) ]
119119 ; CHECK: ret i32 20
120120
121121 %v = lshr i32 %a, 2
130130 define i32 @test9(i32 %a) #0 {
131131 entry:
132132 ; CHECK-LABEL: @test9
133 ; CHECK: call void @llvm.assume
133 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
134134 ; CHECK: ret i32 0
135135
136136 %cmp = icmp sgt i32 %a, 5
144144 define i32 @test10(i32 %a) #0 {
145145 entry:
146146 ; CHECK-LABEL: @test10
147 ; CHECK: call void @llvm.assume
147 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
148148 ; CHECK: ret i32 -2147483648
149149
150150 %cmp = icmp sle i32 %a, -2
158158 define i32 @test11(i32 %a) #0 {
159159 entry:
160160 ; CHECK-LABEL: @test11
161 ; CHECK: call void @llvm.assume
161 ; CHECK: call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
162162 ; CHECK: ret i32 0
163163
164164 %cmp = icmp ule i32 %a, 256
4545 %b = load i32, i32* @B
4646 %b.and = and i32 %b, 1
4747 %b.cnd = icmp eq i32 %b.and, 1
48 call void @llvm.assume(i1 %b.cnd)
48 call void @llvm.assume(i1 %b.cnd) [ "affected"(i32 %b.and, i32 %b) ]
4949
5050 %rhs = add i32 %a, %b
5151 %and = and i32 %a, %rhs
1313 taken:
1414 %res1 = call i8* @escape()
1515 %a = icmp eq i8* %res1, null
16 tail call void @llvm.assume(i1 %a)
16 tail call void @llvm.assume(i1 %a) [ "affected"(i8* %res1) ]
1717 br label %done
1818 not_taken:
1919 %res2 = call i8* @escape()
2020 %b = icmp ne i8* %res2, null
21 tail call void @llvm.assume(i1 %b)
21 tail call void @llvm.assume(i1 %b) [ "affected"(i8* %res2) ]
2222 br label %done
2323
2424 ; An assume that can be used to simplify this comparison dominates each
55 define i32 @test1(i32 %a, i32 %b) #0 {
66 entry:
77 %cmp = icmp sgt i32 %a, 5
8 tail call void @llvm.assume(i1 %cmp)
8 tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
99 %cmp1 = icmp sgt i32 %b, 1234
1010 br i1 %cmp1, label %if.then, label %if.else
1111
3535 define i32 @test2(i32 %a) #0 {
3636 entry:
3737 %cmp = icmp sgt i32 %a, 5
38 tail call void @llvm.assume(i1 %cmp)
38 tail call void @llvm.assume(i1 %cmp) [ "affected"(i32 %a) ]
3939 %cmp1 = icmp sgt i32 %a, 3
4040 br i1 %cmp1, label %if.then, label %return
4141
7474 ; CHECK-LABEL: @reassociate_gep_assume(
7575 ; assume(j >= 0)
7676 %cmp = icmp sgt i32 %j, -1
77 call void @llvm.assume(i1 %cmp)
77 call void @llvm.assume(i1 %cmp) [ "affected"(i32 %j) ]
7878 %1 = add i32 %i, %j
7979 %cmp2 = icmp sgt i32 %1, -1
80 call void @llvm.assume(i1 %cmp2)
80 call void @llvm.assume(i1 %cmp2) [ "affected"(i32 %1) ]
8181
8282 %idxprom.j = zext i32 %j to i64
8383 %2 = getelementptr float, float* %a, i64 %idxprom.j
9090 ; CHECK-LABEL: @test5
9191 ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
9292 %cmp = icmp ult i8 %a, 2
93 call void @llvm.assume(i1 %cmp)
93 call void @llvm.assume(i1 %cmp) [ "affected"(i8 %a) ]
9494 switch i8 %a, label %default [i8 1, label %true
9595 i8 0, label %false]
9696 true:
111111 ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
112112 %and = and i8 %a, 254
113113 %cmp = icmp eq i8 %and, 254
114 call void @llvm.assume(i1 %cmp)
114 call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ]
115115 switch i8 %a, label %default [i8 255, label %true
116116 i8 254, label %false]
117117 true:
133133 ; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
134134 %and = and i8 %a, 254
135135 %cmp = icmp eq i8 %and, 254
136 call void @llvm.assume(i1 %cmp)
136 call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ]
137137 switch i8 %a, label %default [i8 255, label %true
138138 i8 254, label %false
139139 i8 0, label %also_dead]
161161 ; CHECK: switch i8
162162 %and = and i8 %a, 254
163163 %cmp = icmp eq i8 %and, undef
164 call void @llvm.assume(i1 %cmp)
164 call void @llvm.assume(i1 %cmp) [ "affected"(i8 %and, i8 %a) ]
165165 switch i8 %a, label %default [i8 255, label %true
166166 i8 254, label %false]
167167 true: