llvm.org GIT mirror llvm / 1c2d3c5
sched: fix latency of memory dependence chain edges for consistency. For store->load dependencies that may alias, we should always use TrueMemOrderLatency, which may eventually become a subtarget hook. In effect, we should guarantee at least TrueMemOrderLatency on at least one DAG path from a store to a may-alias load. This should fix the standard mode as well as -enable-aa-sched-mi". git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@158380 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 7 years ago
2 changed file(s) with 62 addition(s) and 13 deletion(s). Raw diff Collapse all Expand all
638638 /// checks whether SU can be aliasing any node dominated
639639 /// by it.
640640 static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI,
641 SUnit *SU, SUnit *ExitSU, std::set &CheckList) {
641 SUnit *SU, SUnit *ExitSU, std::set &CheckList,
642 unsigned LatencyToLoad) {
642643 if (!SU)
643644 return;
644645
649650 I != IE; ++I) {
650651 if (SU == *I)
651652 continue;
652 if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr()))
653 (*I)->addPred(SDep(SU, SDep::Order, /*Latency=*/0, /*Reg=*/0,
653 if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr())) {
654 unsigned Latency = ((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0;
655 (*I)->addPred(SDep(SU, SDep::Order, Latency, /*Reg=*/0,
654656 /*isNormalMemory=*/true));
657 }
655658 // Now go through all the chain successors and iterate from them.
656659 // Keep track of visited nodes.
657660 for (SUnit::const_succ_iterator J = (*I)->Succs.begin(),
814817 // after stack slots are lowered to actual addresses.
815818 // TODO: Use an AliasAnalysis and do real alias-analysis queries, and
816819 // produce more precise dependence information.
817 #define STORE_LOAD_LATENCY 1
818 unsigned TrueMemOrderLatency = 0;
820 unsigned TrueMemOrderLatency = MI->mayStore() ? 1 : 0;
819821 if (isGlobalMemoryObject(AA, MI)) {
820822 // Be conservative with these and add dependencies on all memory
821823 // references, even those that are known to not alias.
834836 BarrierChain = SU;
835837 // This is a barrier event that acts as a pivotal node in the DAG,
836838 // so it is safe to clear list of exposed nodes.
837 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes);
839 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes,
840 TrueMemOrderLatency);
838841 RejectMemNodes.clear();
839842 NonAliasMemDefs.clear();
840843 NonAliasMemUses.clear();
842845 // fall-through
843846 new_alias_chain:
844847 // Chain all possibly aliasing memory references though SU.
845 if (AliasChain)
846 addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
848 if (AliasChain) {
849 unsigned ChainLatency = 0;
850 if (AliasChain->getInstr()->mayLoad())
851 ChainLatency = TrueMemOrderLatency;
852 addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes,
853 ChainLatency);
854 }
847855 AliasChain = SU;
848856 for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
849857 addChainDependency(AA, MFI, SU, PendingLoads[k], RejectMemNodes,
857865 addChainDependency(AA, MFI, SU, I->second[i], RejectMemNodes,
858866 TrueMemOrderLatency);
859867 }
860 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes);
868 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes,
869 TrueMemOrderLatency);
861870 PendingLoads.clear();
862871 AliasMemDefs.clear();
863872 AliasMemUses.clear();
864873 } else if (MI->mayStore()) {
865874 bool MayAlias = true;
866 TrueMemOrderLatency = STORE_LOAD_LATENCY;
867875 if (const Value *V = getUnderlyingObjectForInstr(MI, MFI, MayAlias)) {
868876 // A store to a specific PseudoSourceValue. Add precise dependencies.
869877 // Record the def in MemDefs, first adding a dep if there is
904912 addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
905913 // But we also should check dependent instructions for the
906914 // SU in question.
907 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes);
915 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes,
916 TrueMemOrderLatency);
908917 }
909918 // Add dependence on barrier chain, if needed.
910919 // There is no point to check aliasing on barrier event. Even if
926935 /*isArtificial=*/true));
927936 } else if (MI->mayLoad()) {
928937 bool MayAlias = true;
929 TrueMemOrderLatency = 0;
930938 if (MI->isInvariantLoad(AA)) {
931939 // Invariant load, no chain dependencies needed!
932940 } else {
954962 MayAlias = true;
955963 }
956964 if (MayAlias)
957 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes);
965 adjustChainDeps(AA, MFI, SU, &ExitSU, RejectMemNodes, /*Latency=*/0);
958966 // Add dependencies on alias and barrier chains, if needed.
959967 if (MayAlias && AliasChain)
960968 addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
0 ; RUN: llc < %s -o /dev/null "-mtriple=thumbv7-apple-ios" -debug-only=post-RA-sched 2> %t
1 ; RUN: FileCheck %s < %t
2 ; REQUIRES: asserts
3 ; Make sure that mayalias store-load dependencies have one cycle
4 ; latency regardless of whether they are barriers or not.
5
6 ; CHECK: ** List Scheduling
7 ; CHECK: SU(2){{.*}}STR{{.*}}Volatile
8 ; CHECK-NOT: ch SU
9 ; CHECK: ch SU(3): Latency=1
10 ; CHECK-NOT: ch SU
11 ; CHECK: SU(3){{.*}}LDR{{.*}}Volatile
12 ; CHECK-NOT: ch SU
13 ; CHECK: ch SU(2): Latency=1
14 ; CHECK-NOT: ch SU
15 ; CHECK: ** List Scheduling
16 ; CHECK: SU(2){{.*}}STR{{.*}}
17 ; CHECK-NOT: ch SU
18 ; CHECK: ch SU(3): Latency=1
19 ; CHECK-NOT: ch SU
20 ; CHECK: SU(3){{.*}}LDR{{.*}}
21 ; CHECK-NOT: ch SU
22 ; CHECK: ch SU(2): Latency=1
23 ; CHECK-NOT: ch SU
24 define i32 @f1(i32* nocapture %p1, i32* nocapture %p2) nounwind {
25 entry:
26 store volatile i32 65540, i32* %p1, align 4, !tbaa !0
27 %0 = load volatile i32* %p2, align 4, !tbaa !0
28 ret i32 %0
29 }
30
31 define i32 @f2(i32* nocapture %p1, i32* nocapture %p2) nounwind {
32 entry:
33 store i32 65540, i32* %p1, align 4, !tbaa !0
34 %0 = load i32* %p2, align 4, !tbaa !0
35 ret i32 %0
36 }
37
38 !0 = metadata !{metadata !"int", metadata !1}
39 !1 = metadata !{metadata !"omnipotent char", metadata !2}
40 !2 = metadata !{metadata !"Simple C/C++ TBAA"}