llvm.org GIT mirror llvm / c7117c0
[MCA][Scheduler] Change how memory instructions are dispatched to the pending set. NFCI git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@362302 91177308-0d34-0410-b5e6-96231b3b80d8 Andrea Di Biagio 4 months ago
2 changed file(s) with 48 addition(s) and 40 deletion(s). Raw diff Collapse all Expand all
6969 unsigned getNumExecuting() const { return NumExecuting; }
7070 unsigned getNumExecuted() const { return NumExecuted; }
7171
72 const InstRef &getCriticalMemoryInstruction() const {
72 const InstRef &getCriticalMemoryInstruction() const {
7373 return CriticalMemoryInstruction;
7474 }
7575 const CriticalDependency &getCriticalPredecessor() const {
9595 }
9696 bool isReady() const { return NumExecutedPredecessors == NumPredecessors; }
9797 bool isExecuting() const {
98 return NumExecuting == NumInstructions - NumExecuted;
98 return NumExecuting && (NumExecuting == (NumInstructions - NumExecuted));
9999 }
100100 bool isExecuted() const { return NumInstructions == NumExecuted; }
101101
246246 /// Check if a peviously dispatched instruction IR is now ready for execution.
247247 bool isReady(const InstRef &IR) const {
248248 unsigned GroupID = IR.getInstruction()->getLSUTokenID();
249 assert(isValidGroupID(GroupID) &&
250 "Invalid group associated with this instruction!");
251 const MemoryGroup &Group = *Groups.find(GroupID)->second;
249 const MemoryGroup &Group = getGroup(GroupID);
252250 return Group.isReady();
253251 }
254252
255 /// Check if a previously dispatched instruction IR only depends on
256 /// instructions that are currently executing.
253 /// Check if instruction IR only depends on memory instructions that are
254 /// currently executing.
257255 bool isPending(const InstRef &IR) const {
258256 unsigned GroupID = IR.getInstruction()->getLSUTokenID();
259 assert(isValidGroupID(GroupID) &&
260 "Invalid group associated with this instruction!");
261 const MemoryGroup &Group = *Groups.find(GroupID)->second;
257 const MemoryGroup &Group = getGroup(GroupID);
262258 return Group.isPending();
259 }
260
261 /// Check if instruction IR is still waiting on memory operations, and the
262 /// wait time is still unknown.
263 bool isWaiting(const InstRef &IR) const {
264 unsigned GroupID = IR.getInstruction()->getLSUTokenID();
265 const MemoryGroup &Group = getGroup(GroupID);
266 return Group.isWaiting();
267 }
268
269 bool hasDependentUsers(const InstRef &IR) const {
270 unsigned GroupID = IR.getInstruction()->getLSUTokenID();
271 const MemoryGroup &Group = getGroup(GroupID);
272 return !Group.isExecuted() && Group.getNumSuccessors();
263273 }
264274
265275 const MemoryGroup &getGroup(unsigned Index) const {
273283 }
274284
275285 unsigned createMemoryGroup() {
276 Groups.insert(std::make_pair(NextGroupID, llvm::make_unique()));
286 Groups.insert(
287 std::make_pair(NextGroupID, llvm::make_unique()));
277288 return NextGroupID++;
278289 }
279290
103103 SmallVectorImpl &ReadyInstructions) {
104104 const Instruction &Inst = *IR.getInstruction();
105105 bool HasDependentUsers = Inst.hasDependentUsers();
106 HasDependentUsers |= Inst.isMemOp() && LSU.hasDependentUsers(IR);
106107
107108 Resources->releaseBuffers(Inst.getDesc().Buffers);
108109 issueInstructionImpl(IR, UsedResources);
110111 // other dependent instructions. Dependent instructions may be issued during
111112 // this same cycle if operands have ReadAdvance entries. Promote those
112113 // instructions to the ReadySet and notify the caller that those are ready.
113 // If IR is a memory operation, then always call method `promoteToReadySet()`
114 // to notify any dependent memory operations that IR started execution.
115 bool ShouldPromoteInstructions = Inst.isMemOp();
116114 if (HasDependentUsers)
117 ShouldPromoteInstructions |= promoteToPendingSet(PendingInstructions);
118
119 if (ShouldPromoteInstructions)
120 promoteToReadySet(ReadyInstructions);
115 if (promoteToPendingSet(PendingInstructions))
116 promoteToReadySet(ReadyInstructions);
121117 }
122118
123119 bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) {
129125 if (!IR)
130126 break;
131127
128 // Check if there are unsolved register dependencies.
129 Instruction &IS = *IR.getInstruction();
130 if (!IS.isReady() && !IS.updatePending()) {
131 ++I;
132 continue;
133 }
132134 // Check if there are unsolved memory dependencies.
133 Instruction &IS = *IR.getInstruction();
134135 if (IS.isMemOp() && !LSU.isReady(IR)) {
135136 ++I;
136137 continue;
137138 }
138139
139 // Check if there are unsolved register dependencies.
140 if (!IS.isReady() && !IS.updatePending()) {
141 ++I;
142 continue;
143 }
144140 LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR
145141 << " promoted to the READY set.\n");
146142
172168 ++I;
173169 continue;
174170 }
171
172 if (IS.isMemOp() && LSU.isWaiting(IR)) {
173 ++I;
174 continue;
175 }
176
175177 LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR
176178 << " promoted to the PENDING set.\n");
177179
250252 if (Resources->checkAvailability(IS.getDesc()))
251253 continue;
252254
253 if (IS.isMemOp()) {
254 const MemoryGroup &Group = LSU.getGroup(IS.getLSUTokenID());
255 if (Group.isWaiting())
256 continue;
257 if (Group.isPending())
258 MemDeps.emplace_back(IR);
259 }
255 if (IS.isMemOp() && LSU.isPending(IR))
256 MemDeps.emplace_back(IR);
260257
261258 if (IS.isPending())
262259 RegDeps.emplace_back(IR);
308305 if (IS.isMemOp())
309306 IS.setLSUTokenID(LSU.dispatch(IR));
310307
311 if (IS.isPending()) {
308 if (IS.isDispatched() || (IS.isMemOp() && LSU.isWaiting(IR))) {
309 LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n");
310 WaitSet.push_back(IR);
311 return false;
312 }
313
314 if (IS.isPending() || (IS.isMemOp() && LSU.isPending(IR))) {
312315 LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR
313316 << " to the PendingSet\n");
314317 PendingSet.push_back(IR);
316319 return false;
317320 }
318321
319 // Memory operations that still have unsolved memory dependencies are
320 // initially dispatched to the WaitSet.
321 if (!IS.isReady() || (IS.isMemOp() && !LSU.isReady(IR))) {
322 LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n");
323 WaitSet.push_back(IR);
324 return false;
325 }
326
322 assert(IS.isReady() && (!IS.isMemOp() || LSU.isReady(IR)) &&
323 "Unexpected internal state found!");
327324 // Don't add a zero-latency instruction to the Ready queue.
328325 // A zero-latency instruction doesn't consume any scheduler resources. That is
329326 // because it doesn't need to be executed, and it is often removed at register