llvm.org GIT mirror llvm / a78d322
ScheduleDAG interface. Added OrderKind to distinguish nonregister dependencies. This is in preparation for adding "weak" DAG edges, but generally simplifies the design. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@167435 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 7 years ago
6 changed file(s) with 93 addition(s) and 98 deletion(s). Raw diff Collapse all Expand all
5151 Order ///< Any other ordering dependency.
5252 };
5353
54 enum OrderKind {
55 Barrier, ///< An unknown scheduling barrier.
56 MayAliasMem, ///< Nonvolatile load/Store instructions that may alias.
57 MustAliasMem, ///< Nonvolatile load/Store instructions that must alias.
58 Artificial ///< Arbitrary weak DAG edge (no actual dependence).
59 };
60
5461 private:
5562 /// Dep - A pointer to the depending/depended-on SUnit, and an enum
5663 /// indicating the kind of the dependency.
6471 unsigned Reg;
6572
6673 /// Order - Additional information about Order dependencies.
67 struct {
68 /// isNormalMemory - True if both sides of the dependence
69 /// access memory in non-volatile and fully modeled ways.
70 bool isNormalMemory : 1;
71
72 /// isMustAlias - True if both sides of the dependence are known to
73 /// access the same memory.
74 bool isMustAlias : 1;
75
76 /// isArtificial - True if this is an artificial dependency, meaning
77 /// it is not necessary for program correctness, and may be safely
78 /// deleted if necessary.
79 bool isArtificial : 1;
80 } Order;
74 unsigned OrdKind; // enum OrderKind
8175 } Contents;
8276
8377 /// Latency - The time associated with this edge. Often this is just
8579 /// models may provide additional information about specific edges.
8680 unsigned Latency;
8781 /// Record MinLatency seperately from "expected" Latency.
82 ///
83 /// FIXME: this field is not packed on LP64. Convert to 16-bit DAG edge
84 /// latency after introducing saturating truncation.
8885 unsigned MinLatency;
8986
9087 public:
9491 SDep() : Dep(0, Data) {}
9592
9693 /// SDep - Construct an SDep with the specified values.
97 SDep(SUnit *S, Kind kind, unsigned latency = 1, unsigned Reg = 0,
98 bool isNormalMemory = false, bool isMustAlias = false,
99 bool isArtificial = false)
100 : Dep(S, kind), Contents(), Latency(latency), MinLatency(latency) {
94 SDep(SUnit *S, Kind kind, unsigned Reg)
95 : Dep(S, kind), Contents() {
10196 switch (kind) {
97 default:
98 llvm_unreachable("Reg given for non-register dependence!");
10299 case Anti:
103100 case Output:
104101 assert(Reg != 0 &&
105102 "SDep::Anti and SDep::Output must use a non-zero Reg!");
106 // fall through
103 Contents.Reg = Reg;
104 Latency = 0;
105 break;
107106 case Data:
108 assert(!isMustAlias && "isMustAlias only applies with SDep::Order!");
109 assert(!isArtificial && "isArtificial only applies with SDep::Order!");
110107 Contents.Reg = Reg;
111 break;
112 case Order:
113 assert(Reg == 0 && "Reg given for non-register dependence!");
114 Contents.Order.isNormalMemory = isNormalMemory;
115 Contents.Order.isMustAlias = isMustAlias;
116 Contents.Order.isArtificial = isArtificial;
108 Latency = 1;
117109 break;
118110 }
111 MinLatency = Latency;
112 }
113 SDep(SUnit *S, OrderKind kind)
114 : Dep(S, Order), Contents(), Latency(0), MinLatency(0) {
115 Contents.OrdKind = kind;
119116 }
120117
121118 /// Return true if the specified SDep is equivalent except for latency.
127124 case Output:
128125 return Contents.Reg == Other.Contents.Reg;
129126 case Order:
130 return Contents.Order.isNormalMemory ==
131 Other.Contents.Order.isNormalMemory &&
132 Contents.Order.isMustAlias == Other.Contents.Order.isMustAlias &&
133 Contents.Order.isArtificial == Other.Contents.Order.isArtificial;
127 return Contents.OrdKind == Other.Contents.OrdKind;
134128 }
135129 llvm_unreachable("Invalid dependency kind!");
136130 }
193187 /// memory accesses where both sides of the dependence access memory
194188 /// in non-volatile and fully modeled ways.
195189 bool isNormalMemory() const {
196 return getKind() == Order && Contents.Order.isNormalMemory;
190 return getKind() == Order && (Contents.OrdKind == MayAliasMem
191 || Contents.OrdKind == MustAliasMem);
197192 }
198193
199194 /// isMustAlias - Test if this is an Order dependence that is marked
200195 /// as "must alias", meaning that the SUnits at either end of the edge
201196 /// have a memory dependence on a known memory location.
202197 bool isMustAlias() const {
203 return getKind() == Order && Contents.Order.isMustAlias;
198 return getKind() == Order && Contents.OrdKind == MustAliasMem;
204199 }
205200
206201 /// isArtificial - Test if this is an Order dependence that is marked
207202 /// as "artificial", meaning it isn't necessary for correctness.
208203 bool isArtificial() const {
209 return getKind() == Order && Contents.Order.isArtificial;
204 return getKind() == Order && Contents.OrdKind == Artificial;
210205 }
211206
212207 /// isAssignedRegDep - Test if this is a Data dependence that is
244244 if (UseSU == SU)
245245 continue;
246246
247 SDep dep(SU, SDep::Data, 1, *Alias);
247 SDep dep(SU, SDep::Data, *Alias);
248248
249249 // Adjust the dependence latency using operand def/use information,
250250 // then allow the target to perform its own adjustments.
290290 (Kind != SDep::Output || !MO.isDead() ||
291291 !DefSU->getInstr()->registerDefIsDead(*Alias))) {
292292 if (Kind == SDep::Anti)
293 DefSU->addPred(SDep(SU, Kind, 0, /*Reg=*/*Alias));
293 DefSU->addPred(SDep(SU, Kind, /*Reg=*/*Alias));
294294 else {
295 unsigned AOLat =
295 SDep Dep(SU, Kind, /*Reg=*/*Alias);
296 unsigned OutLatency =
296297 SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
297 DefSU->addPred(SDep(SU, Kind, AOLat, /*Reg=*/*Alias));
298 Dep.setMinLatency(OutLatency);
299 Dep.setLatency(OutLatency);
300 DefSU->addPred(Dep);
298301 }
299302 }
300303 }
363366 else {
364367 SUnit *DefSU = DefI->SU;
365368 if (DefSU != SU && DefSU != &ExitSU) {
369 SDep Dep(SU, SDep::Output, Reg);
366370 unsigned OutLatency =
367371 SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
368 DefSU->addPred(SDep(SU, SDep::Output, OutLatency, Reg));
372 Dep.setMinLatency(OutLatency);
373 Dep.setLatency(OutLatency);
374 DefSU->addPred(Dep);
369375 }
370376 DefI->SU = SU;
371377 }
395401 if (DefSU) {
396402 // The reaching Def lives within this scheduling region.
397403 // Create a data dependence.
398 SDep dep(DefSU, SDep::Data, 1, Reg);
404 SDep dep(DefSU, SDep::Data, Reg);
399405 // Adjust the dependence latency using operand def/use information, then
400406 // allow the target to perform its own adjustments.
401407 int DefOp = Def->findRegisterDefOperandIdx(Reg);
413419 // Add antidependence to the following def of the vreg it uses.
414420 VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg);
415421 if (DefI != VRegDefs.end() && DefI->SU != SU)
416 DefI->SU->addPred(SDep(SU, SDep::Anti, 0, Reg));
422 DefI->SU->addPred(SDep(SU, SDep::Anti, Reg));
417423 }
418424
419425 /// Return true if MI is an instruction we are unable to reason about
553559 // and stop descending.
554560 if (*Depth > 200 ||
555561 MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
556 SUb->addPred(SDep(SUa, SDep::Order, /*Latency=*/0, /*Reg=*/0,
557 /*isNormalMemory=*/true));
562 SUb->addPred(SDep(SUa, SDep::MayAliasMem));
558563 return *Depth;
559564 }
560565 // Track current depth.
585590 if (SU == *I)
586591 continue;
587592 if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr())) {
588 unsigned Latency = ((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0;
589 (*I)->addPred(SDep(SU, SDep::Order, Latency, /*Reg=*/0,
590 /*isNormalMemory=*/true));
593 SDep Dep(SU, SDep::MayAliasMem);
594 Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0);
595 (*I)->addPred(Dep);
591596 }
592597 // Now go through all the chain successors and iterate from them.
593598 // Keep track of visited nodes.
610615 // If this is a false dependency,
611616 // do not add the edge, but rememeber the rejected node.
612617 if (!EnableAASchedMI ||
613 MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr()))
614 SUb->addPred(SDep(SUa, SDep::Order, TrueMemOrderLatency, /*Reg=*/0,
615 isNormalMemory));
618 MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
619 SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier);
620 Dep.setLatency(TrueMemOrderLatency);
621 SUb->addPred(Dep);
622 }
616623 else {
617624 // Duplicate entries should be ignored.
618625 RejectList.insert(SUb);
754761 // references, even those that are known to not alias.
755762 for (std::map::iterator I =
756763 NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) {
757 I->second->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
764 I->second->addPred(SDep(SU, SDep::Barrier));
758765 }
759766 for (std::map >::iterator I =
760767 NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) {
761 for (unsigned i = 0, e = I->second.size(); i != e; ++i)
762 I->second[i]->addPred(SDep(SU, SDep::Order, TrueMemOrderLatency));
768 for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
769 SDep Dep(SU, SDep::Barrier);
770 Dep.setLatency(TrueMemOrderLatency);
771 I->second[i]->addPred(Dep);
772 }
763773 }
764774 // Add SU to the barrier chain.
765775 if (BarrierChain)
766 BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
776 BarrierChain->addPred(SDep(SU, SDep::Barrier));
767777 BarrierChain = SU;
768778 // This is a barrier event that acts as a pivotal node in the DAG,
769779 // so it is safe to clear list of exposed nodes.
851861 // SU and barrier _could_ be reordered, they should not. In addition,
852862 // we have lost all RejectMemNodes below barrier.
853863 if (BarrierChain)
854 BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
864 BarrierChain->addPred(SDep(SU, SDep::Barrier));
855865 } else {
856866 // Treat all other stores conservatively.
857867 goto new_alias_chain;
860870 if (!ExitSU.isPred(SU))
861871 // Push store's up a bit to avoid them getting in between cmp
862872 // and branches.
863 ExitSU.addPred(SDep(SU, SDep::Order, 0,
864 /*Reg=*/0, /*isNormalMemory=*/false,
865 /*isMustAlias=*/false,
866 /*isArtificial=*/true));
873 ExitSU.addPred(SDep(SU, SDep::Artificial));
867874 } else if (MI->mayLoad()) {
868875 bool MayAlias = true;
869876 if (MI->isInvariantLoad(AA)) {
898905 if (MayAlias && AliasChain)
899906 addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
900907 if (BarrierChain)
901 BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
908 BarrierChain->addPred(SDep(SU, SDep::Barrier));
902909 }
903910 }
904911 }
335335 }
336336 }
337337 if (isNewLoad) {
338 AddPred(NewSU, SDep(LoadSU, SDep::Order, LoadSU->Latency));
338 SDep D(LoadSU, SDep::Barrier);
339 D.setLatency(LoadSU->Latency);
340 AddPred(NewSU, D);
339341 }
340342
341343 ++NumUnfolds;
411413 for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
412414 RemovePred(DelDeps[i].first, DelDeps[i].second);
413415 }
414
415 AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
416 AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
416 SDep FromDep(SU, SDep::Data, Reg);
417 FromDep.setLatency(SU->Latency);
418 AddPred(CopyFromSU, FromDep);
419 SDep ToDep(CopyFromSU, SDep::Data, 0);
420 ToDep.setLatency(CopyFromSU->Latency);
421 AddPred(CopyToSU, ToDep);
417422
418423 Copies.push_back(CopyFromSU);
419424 Copies.push_back(CopyToSU);
590595 InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
591596 DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
592597 << " to SU #" << Copies.front()->NodeNum << "\n");
593 AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
594 /*Reg=*/0, /*isNormalMemory=*/false,
595 /*isMustAlias=*/false, /*isArtificial=*/true));
598 AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
596599 NewDef = Copies.back();
597600 }
598601
599602 DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
600603 << " to SU #" << TrySU->NodeNum << "\n");
601604 LiveRegDefs[Reg] = NewDef;
602 AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
603 /*Reg=*/0, /*isNormalMemory=*/false,
604 /*isMustAlias=*/false, /*isArtificial=*/true));
605 AddPred(NewDef, SDep(TrySU, SDep::Artificial));
605606 TrySU->isAvailable = false;
606607 CurSU = NewDef;
607608 }
10571057
10581058 // Add a data dependency to reflect that NewSU reads the value defined
10591059 // by LoadSU.
1060 AddPred(NewSU, SDep(LoadSU, SDep::Data, LoadSU->Latency));
1060 SDep D(LoadSU, SDep::Data, 0);
1061 D.setLatency(LoadSU->Latency);
1062 AddPred(NewSU, D);
10611063
10621064 if (isNewLoad)
10631065 AvailableQueue->addNode(LoadSU);
11391141 // Avoid scheduling the def-side copy before other successors. Otherwise
11401142 // we could introduce another physreg interference on the copy and
11411143 // continue inserting copies indefinitely.
1142 SDep D(CopyFromSU, SDep::Order, /*Latency=*/0,
1143 /*Reg=*/0, /*isNormalMemory=*/false,
1144 /*isMustAlias=*/false, /*isArtificial=*/true);
1145 AddPred(SuccSU, D);
1144 AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial));
11461145 }
11471146 }
11481147 for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
11491148 RemovePred(DelDeps[i].first, DelDeps[i].second);
11501149
1151 AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
1152 AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
1150 SDep FromDep(SU, SDep::Data, Reg);
1151 FromDep.setLatency(SU->Latency);
1152 AddPred(CopyFromSU, FromDep);
1153 SDep ToDep(CopyFromSU, SDep::Data, 0);
1154 ToDep.setLatency(CopyFromSU->Latency);
1155 AddPred(CopyToSU, ToDep);
11531156
11541157 AvailableQueue->updateNode(SU);
11551158 AvailableQueue->addNode(CopyFromSU);
13581361 if (!BtSU->isPending)
13591362 AvailableQueue->remove(BtSU);
13601363 }
1361 AddPred(TrySU, SDep(BtSU, SDep::Order, /*Latency=*/1,
1362 /*Reg=*/0, /*isNormalMemory=*/false,
1363 /*isMustAlias=*/false, /*isArtificial=*/true));
1364 AddPred(TrySU, SDep(BtSU, SDep::Artificial));
13641365
13651366 // If one or more successors has been unscheduled, then the current
13661367 // node is no longer avaialable. Schedule a successor that's now
14121413 InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
14131414 DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum
14141415 << " to SU #" << Copies.front()->NodeNum << "\n");
1415 AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
1416 /*Reg=*/0, /*isNormalMemory=*/false,
1417 /*isMustAlias=*/false,
1418 /*isArtificial=*/true));
1416 AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
14191417 NewDef = Copies.back();
14201418 }
14211419
14221420 DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum
14231421 << " to SU #" << TrySU->NodeNum << "\n");
14241422 LiveRegDefs[Reg] = NewDef;
1425 AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
1426 /*Reg=*/0, /*isNormalMemory=*/false,
1427 /*isMustAlias=*/false,
1428 /*isArtificial=*/true));
1423 AddPred(NewDef, SDep(TrySU, SDep::Artificial));
14291424 TrySU->isAvailable = false;
14301425 CurSU = NewDef;
14311426 }
29352930 !scheduleDAG->IsReachable(SuccSU, SU)) {
29362931 DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #"
29372932 << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
2938 scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Order, /*Latency=*/0,
2939 /*Reg=*/0, /*isNormalMemory=*/false,
2940 /*isMustAlias=*/false,
2941 /*isArtificial=*/true));
2933 scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial));
29422934 }
29432935 }
29442936 }
484484 if(isChain && OpN->getOpcode() == ISD::TokenFactor)
485485 OpLatency = 0;
486486
487 const SDep &dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data,
488 OpLatency, PhysReg);
487 SDep Dep = isChain ? SDep(OpSU, SDep::Barrier)
488 : SDep(OpSU, SDep::Data, PhysReg);
489 Dep.setLatency(OpLatency);
489490 if (!isChain && !UnitLatencies) {
490 computeOperandLatency(OpN, N, i, const_cast(dep));
491 ST.adjustSchedDependency(OpSU, SU, const_cast(dep));
491 computeOperandLatency(OpN, N, i, Dep);
492 ST.adjustSchedDependency(OpSU, SU, Dep);
492493 }
493494
494 if (!SU->addPred(dep) && !dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
495 if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
495496 // Multiple register uses are combined in the same SUnit. For example,
496497 // we could have a set of glued nodes with all their defs consumed by
497498 // another set of glued nodes. Register pressure tracking sees this as
3030 LastSequentialCall = &(SUnits[su]);
3131 // Look for a compare that defines a predicate.
3232 else if (SUnits[su].getInstr()->isCompare() && LastSequentialCall)
33 SUnits[su].addPred(SDep(LastSequentialCall, SDep::Order, 0, /*Reg=*/0,
34 false));
33 SUnits[su].addPred(SDep(LastSequentialCall, SDep::Barrier));
3534 }
3635 }
3736