llvm.org GIT mirror llvm / 57902cc
[PBQP] Unique allowed-sets for nodes in the PBQP graph and use pairs of these sets as keys into a cache of interference matrice values in the Interference constraint adder. Creating interference matrices was one of the large remaining time-sinks in PBQP. Caching them reduces the total compile time (when using PBQP) on the nightly test suite by ~10%. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@220688 91177308-0d34-0410-b5e6-96231b3b80d8 Lang Hames 5 years ago
5 changed file(s) with 250 addition(s) and 89 deletion(s). Raw diff Collapse all Expand all
2727 template
2828 class ValuePool {
2929 public:
30 typedef std::shared_ptr<ValueT> PoolRef;
30 typedef std::shared_ptr<const ValueT> PoolRef;
3131
3232 private:
3333
3737 PoolEntry(ValuePool &Pool, ValueKeyT Value)
3838 : Pool(Pool), Value(std::move(Value)) {}
3939 ~PoolEntry() { Pool.removeEntry(this); }
40 ValueT& getValue() { return Value; }
4140 const ValueT& getValue() const { return Value; }
4241 private:
4342 ValuePool &Pool;
386386 return NId;
387387 }
388388
389 /// @brief Add a node bypassing the cost allocator.
390 /// @param Costs Cost vector ptr for the new node (must be convertible to
391 /// VectorPtr).
392 /// @return Node iterator for the added node.
393 ///
394 /// This method allows for fast addition of a node whose costs don't need
395 /// to be passed through the cost allocator. The most common use case for
396 /// this is when duplicating costs from an existing node (when using a
397 /// pooling allocator). These have already been uniqued, so we can avoid
398 /// re-constructing and re-uniquing them by attaching them directly to the
399 /// new node.
400 template
401 NodeId addNodeBypassingCostAllocator(OtherVectorPtrT Costs) {
402 NodeId NId = addConstructedNode(NodeEntry(Costs));
403 if (Solver)
404 Solver->handleAddNode(NId);
405 return NId;
406 }
407
389408 /// @brief Add an edge between the given nodes with the given costs.
390409 /// @param N1Id First node.
391410 /// @param N2Id Second node.
411 /// @param Costs Cost matrix for new edge.
392412 /// @return Edge iterator for the added edge.
393413 template
394414 EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs) {
403423 return EId;
404424 }
405425
426 /// @brief Add an edge bypassing the cost allocator.
427 /// @param N1Id First node.
428 /// @param N2Id Second node.
429 /// @param Costs Cost matrix for new edge.
430 /// @return Edge iterator for the added edge.
431 ///
432 /// This method allows for fast addition of an edge whose costs don't need
433 /// to be passed through the cost allocator. The most common use case for
434 /// this is when duplicating costs from an existing edge (when using a
435 /// pooling allocator). These have already been uniqued, so we can avoid
436 /// re-constructing and re-uniquing them by attaching them directly to the
437 /// new edge.
438 template
439 NodeId addEdgeBypassingCostAllocator(NodeId N1Id, NodeId N2Id,
440 OtherMatrixPtrT Costs) {
441 assert(getNodeCosts(N1Id).getLength() == Costs->getRows() &&
442 getNodeCosts(N2Id).getLength() == Costs->getCols() &&
443 "Matrix dimensions mismatch.");
444 // Get cost matrix from the problem domain.
445 EdgeId EId = addConstructedEdge(EdgeEntry(N1Id, N2Id, Costs));
446 if (Solver)
447 Solver->handleAddEdge(EId);
448 return EId;
449 }
450
406451 /// @brief Returns true if the graph is empty.
407452 bool empty() const { return NodeIdSet(*this).empty(); }
408453
430475 getNode(NId).Costs = AllocatedCosts;
431476 }
432477
433 /// @brief Get a node's cost vector (const version).
478 /// @brief Get a VectorPtr to a node's cost vector. Rarely useful - use
479 /// getNodeCosts where possible.
480 /// @param NId Node id.
481 /// @return VectorPtr to node cost vector.
482 ///
483 /// This method is primarily useful for duplicating costs quickly by
484 /// bypassing the cost allocator. See addNodeBypassingCostAllocator. Prefer
485 /// getNodeCosts when dealing with node cost values.
486 const VectorPtr& getNodeCostsPtr(NodeId NId) const {
487 return getNode(NId).Costs;
488 }
489
490 /// @brief Get a node's cost vector.
434491 /// @param NId Node id.
435492 /// @return Node cost vector.
436 const Vector& getNodeCosts(NodeId NId) const { return *getNode(NId).Costs; }
493 const Vector& getNodeCosts(NodeId NId) const {
494 return *getNodeCostsPtr(NId);
495 }
437496
438497 NodeMetadata& getNodeMetadata(NodeId NId) {
439498 return getNode(NId).Metadata;
458517 getEdge(EId).Costs = AllocatedCosts;
459518 }
460519
461 /// @brief Get an edge's cost matrix (const version).
520 /// @brief Get a MatrixPtr to a node's cost matrix. Rarely useful - use
521 /// getEdgeCosts where possible.
522 /// @param EId Edge id.
523 /// @return MatrixPtr to edge cost matrix.
524 ///
525 /// This method is primarily useful for duplicating costs quickly by
526 /// bypassing the cost allocator. See addNodeBypassingCostAllocator. Prefer
527 /// getEdgeCosts when dealing with edge cost values.
528 const MatrixPtr& getEdgeCostsPtr(EdgeId EId) const {
529 return getEdge(EId).Costs;
530 }
531
532 /// @brief Get an edge's cost matrix.
462533 /// @param EId Edge id.
463534 /// @return Edge cost matrix.
464535 const Matrix& getEdgeCosts(EdgeId EId) const {
465536 return *getEdge(EId).Costs;
466537 }
467538
468 EdgeMetadata& getEdgeMetadata(EdgeId NId) {
469 return getEdge(NId).Metadata;
470 }
471
472 const EdgeMetadata& getEdgeMetadata(EdgeId NId) const {
473 return getEdge(NId).Metadata;
539 EdgeMetadata& getEdgeMetadata(EdgeId EId) {
540 return getEdge(EId).Metadata;
541 }
542
543 const EdgeMetadata& getEdgeMetadata(EdgeId EId) const {
544 return getEdge(EId).Metadata;
474545 }
475546
476547 /// @brief Get the first node connected to this edge.
7272 std::unique_ptr UnsafeCols;
7373 };
7474
75 /// \brief Holds a vector of the allowed physical regs for a vreg.
76 class AllowedRegVector {
77 friend hash_code hash_value(const AllowedRegVector &);
78 public:
79
80 AllowedRegVector() : NumOpts(0), Opts(nullptr) {}
81
82 AllowedRegVector(const std::vector &OptVec)
83 : NumOpts(OptVec.size()), Opts(new unsigned[NumOpts]) {
84 std::copy(OptVec.begin(), OptVec.end(), Opts.get());
85 }
86
87 AllowedRegVector(const AllowedRegVector &Other)
88 : NumOpts(Other.NumOpts), Opts(new unsigned[NumOpts]) {
89 std::copy(Other.Opts.get(), Other.Opts.get() + NumOpts, Opts.get());
90 }
91
92 AllowedRegVector(AllowedRegVector &&Other)
93 : NumOpts(std::move(Other.NumOpts)), Opts(std::move(Other.Opts)) {}
94
95 AllowedRegVector& operator=(const AllowedRegVector &Other) {
96 NumOpts = Other.NumOpts;
97 Opts.reset(new unsigned[NumOpts]);
98 std::copy(Other.Opts.get(), Other.Opts.get() + NumOpts, Opts.get());
99 return *this;
100 }
101
102 AllowedRegVector& operator=(AllowedRegVector &&Other) {
103 NumOpts = std::move(Other.NumOpts);
104 Opts = std::move(Other.Opts);
105 return *this;
106 }
107
108 unsigned size() const { return NumOpts; }
109 unsigned operator[](size_t I) const { return Opts[I]; }
110
111 bool operator==(const AllowedRegVector &Other) const {
112 if (NumOpts != Other.NumOpts)
113 return false;
114 return std::equal(Opts.get(), Opts.get() + NumOpts, Other.Opts.get());
115 }
116
117 bool operator!=(const AllowedRegVector &Other) const {
118 return !(*this == Other);
119 }
120
121 private:
122 unsigned NumOpts;
123 std::unique_ptr Opts;
124 };
125
126 inline hash_code hash_value(const AllowedRegVector &OptRegs) {
127 unsigned *OStart = OptRegs.Opts.get();
128 unsigned *OEnd = OptRegs.Opts.get() + OptRegs.NumOpts;
129 return hash_combine(OptRegs.NumOpts,
130 hash_combine_range(OStart, OEnd));
131 }
132
133 /// \brief Holds graph-level metadata relevent to PBQP RA problems.
134 class GraphMetadata {
135 private:
136 typedef ValuePool AllowedRegVecPool;
137 public:
138
139 typedef typename AllowedRegVecPool::PoolRef AllowedRegVecRef;
140
141 GraphMetadata(MachineFunction &MF,
142 LiveIntervals &LIS,
143 MachineBlockFrequencyInfo &MBFI)
144 : MF(MF), LIS(LIS), MBFI(MBFI) {}
145
146 MachineFunction &MF;
147 LiveIntervals &LIS;
148 MachineBlockFrequencyInfo &MBFI;
149
150 void setNodeIdForVReg(unsigned VReg, GraphBase::NodeId NId) {
151 VRegToNodeId[VReg] = NId;
152 }
153
154 GraphBase::NodeId getNodeIdForVReg(unsigned VReg) const {
155 auto VRegItr = VRegToNodeId.find(VReg);
156 if (VRegItr == VRegToNodeId.end())
157 return GraphBase::invalidNodeId();
158 return VRegItr->second;
159 }
160
161 void eraseNodeIdForVReg(unsigned VReg) {
162 VRegToNodeId.erase(VReg);
163 }
164
165 AllowedRegVecRef getAllowedRegs(AllowedRegVector Allowed) {
166 return AllowedRegVecs.getValue(std::move(Allowed));
167 }
168
169 private:
170 DenseMap VRegToNodeId;
171 AllowedRegVecPool AllowedRegVecs;
172 };
173
174 /// \brief Holds solver state and other metadata relevant to each PBQP RA node.
75175 class NodeMetadata {
76176 public:
77 typedef std::vector OptionToRegMap;
177 typedef AllowedRegVector AllowedRegVector;
78178
79179 typedef enum { Unprocessed,
80180 OptimallyReducible,
90190 NodeMetadata(const NodeMetadata &Other)
91191 : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts),
92192 OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg),
93 OptionRegs(Other.OptionRegs) {
193 AllowedRegs(Other.AllowedRegs) {
94194 std::copy(&Other.OptUnsafeEdges[0], &Other.OptUnsafeEdges[NumOpts],
95195 &OptUnsafeEdges[0]);
96196 }
100200 NodeMetadata(NodeMetadata &&Other)
101201 : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts),
102202 OptUnsafeEdges(std::move(Other.OptUnsafeEdges)), VReg(Other.VReg),
103 OptionRegs(std::move(Other.OptionRegs)) {}
203 AllowedRegs(std::move(Other.AllowedRegs)) {}
104204
105205 // FIXME: Re-implementing default behavior to work around MSVC. Remove once
106206 // MSVC synthesizes move constructors properly.
112212 std::copy(Other.OptUnsafeEdges.get(), Other.OptUnsafeEdges.get() + NumOpts,
113213 OptUnsafeEdges.get());
114214 VReg = Other.VReg;
115 OptionRegs = Other.OptionRegs;
215 AllowedRegs = Other.AllowedRegs;
116216 return *this;
117217 }
118218
124224 DeniedOpts = Other.DeniedOpts;
125225 OptUnsafeEdges = std::move(Other.OptUnsafeEdges);
126226 VReg = Other.VReg;
127 OptionRegs = std::move(Other.OptionRegs);
227 AllowedRegs = std::move(Other.AllowedRegs);
128228 return *this;
129229 }
130230
131231 void setVReg(unsigned VReg) { this->VReg = VReg; }
132232 unsigned getVReg() const { return VReg; }
133233
134 void setOptionRegs(OptionToRegMap OptionRegs) {
135 this->OptionRegs = std::move(OptionRegs);
136 }
137 const OptionToRegMap& getOptionRegs() const { return OptionRegs; }
234 void setAllowedRegs(GraphMetadata::AllowedRegVecRef AllowedRegs) {
235 this->AllowedRegs = std::move(AllowedRegs);
236 }
237 const AllowedRegVector& getAllowedRegs() const { return *AllowedRegs; }
138238
139239 void setup(const Vector& Costs) {
140240 NumOpts = Costs.getLength() - 1;
172272 unsigned DeniedOpts;
173273 std::unique_ptr OptUnsafeEdges;
174274 unsigned VReg;
175 OptionToRegMap OptionRegs;
275 GraphMetadata::AllowedRegVecRef AllowedRegs;
176276 };
177277
178278 class RegAllocSolverImpl {
189289 typedef GraphBase::EdgeId EdgeId;
190290
191291 typedef RegAlloc::NodeMetadata NodeMetadata;
192
193292 struct EdgeMetadata { };
194
195 class GraphMetadata {
196 public:
197 GraphMetadata(MachineFunction &MF,
198 LiveIntervals &LIS,
199 MachineBlockFrequencyInfo &MBFI)
200 : MF(MF), LIS(LIS), MBFI(MBFI) {}
201
202 MachineFunction &MF;
203 LiveIntervals &LIS;
204 MachineBlockFrequencyInfo &MBFI;
205
206 void setNodeIdForVReg(unsigned VReg, GraphBase::NodeId NId) {
207 VRegToNodeId[VReg] = NId;
208 }
209
210 GraphBase::NodeId getNodeIdForVReg(unsigned VReg) const {
211 auto VRegItr = VRegToNodeId.find(VReg);
212 if (VRegItr == VRegToNodeId.end())
213 return GraphBase::invalidNodeId();
214 return VRegItr->second;
215 }
216
217 void eraseNodeIdForVReg(unsigned VReg) {
218 VRegToNodeId.erase(VReg);
219 }
220
221 private:
222 DenseMap VRegToNodeId;
223 };
293 typedef RegAlloc::GraphMetadata GraphMetadata;
224294
225295 typedef PBQP::Graph Graph;
226296
165165 class Interference : public PBQPRAConstraint {
166166 private:
167167
168 private:
169
170 typedef const PBQP::RegAlloc::AllowedRegVector* AllowedRegVecPtr;
171 typedef std::pair IMatrixKey;
172 typedef DenseMap IMatrixCache;
173
168174 // Holds (Interval, CurrentSegmentID, and NodeId). The first two are required
169175 // for the fast interference graph construction algorithm. The last is there
170176 // to save us from looking up node ids via the VRegToNode map in the graph
225231 // number of registers, but rather the size of the largest clique in the
226232 // graph. Still, we expect this to be better than N^2.
227233 LiveIntervals &LIS = G.getMetadata().LIS;
228 const TargetRegisterInfo &TRI =
229 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
234
235 // Interferenc matrices are incredibly regular - they're only a function of
236 // the allowed sets, so we cache them to avoid the overhead of constructing
237 // and uniquing them.
238 IMatrixCache C;
230239
231240 typedef std::set IntervalSet;
232241 typedef std::priority_queue,
274283 // Check that we haven't already added this edge
275284 // FIXME: findEdge is expensive in the worst case (O(max_clique(G))).
276285 // It might be better to replace this with a local bit-matrix.
277 if (G.findEdge(NId, MId) != PBQP::GraphBase::invalidEdgeId())
286 if (G.findEdge(NId, MId) != PBQPRAGraph::invalidEdgeId())
278287 continue;
279288
280289 // This is a new edge - add it to the graph.
281 const auto &NOpts = G.getNodeMetadata(NId).getOptionRegs();
282 const auto &MOpts = G.getNodeMetadata(MId).getOptionRegs();
283 G.addEdge(NId, MId, createInterferenceMatrix(TRI, NOpts, MOpts));
290 createInterferenceEdge(G, NId, MId, C);
284291 }
285292
286293 // Finally, add Cur to the Active set.
290297
291298 private:
292299
293 PBQPRAGraph::RawMatrix createInterferenceMatrix(
294 const TargetRegisterInfo &TRI,
295 const PBQPRAGraph::NodeMetadata::OptionToRegMap &NOpts,
296 const PBQPRAGraph::NodeMetadata::OptionToRegMap &MOpts) {
297 PBQPRAGraph::RawMatrix M(NOpts.size() + 1, MOpts.size() + 1, 0);
298 for (unsigned I = 0; I != NOpts.size(); ++I) {
299 unsigned PRegN = NOpts[I];
300 for (unsigned J = 0; J != MOpts.size(); ++J) {
301 unsigned PRegM = MOpts[J];
300 void createInterferenceEdge(PBQPRAGraph &G, PBQPRAGraph::NodeId NId,
301 PBQPRAGraph::NodeId MId, IMatrixCache &C) {
302
303 const TargetRegisterInfo &TRI =
304 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
305
306 const auto &NRegs = G.getNodeMetadata(NId).getAllowedRegs();
307 const auto &MRegs = G.getNodeMetadata(MId).getAllowedRegs();
308
309 // Try looking the edge costs up in the IMatrixCache first.
310 IMatrixKey K(&NRegs, &MRegs);
311 IMatrixCache::iterator I = C.find(K);
312 if (I != C.end()) {
313 G.addEdgeBypassingCostAllocator(NId, MId, I->second);
314 return;
315 }
316
317 PBQPRAGraph::RawMatrix M(NRegs.size() + 1, MRegs.size() + 1, 0);
318 for (unsigned I = 0; I != NRegs.size(); ++I) {
319 unsigned PRegN = NRegs[I];
320 for (unsigned J = 0; J != MRegs.size(); ++J) {
321 unsigned PRegM = MRegs[J];
302322 if (TRI.regsOverlap(PRegN, PRegM))
303323 M[I + 1][J + 1] = std::numeric_limits::infinity();
304324 }
305325 }
306326
307 return M;
327 PBQPRAGraph::EdgeId EId = G.addEdge(NId, MId, std::move(M));
328 C[K] = G.getEdgeCostsPtr(EId);
308329 }
309330 };
310331
340361
341362 PBQPRAGraph::NodeId NId = G.getMetadata().getNodeIdForVReg(SrcReg);
342363
343 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed =
344 G.getNodeMetadata(NId).getOptionRegs();
364 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed =
365 G.getNodeMetadata(NId).getAllowedRegs();
345366
346367 unsigned PRegOpt = 0;
347368 while (PRegOpt < Allowed.size() && Allowed[PRegOpt] != DstReg)
355376 } else {
356377 PBQPRAGraph::NodeId N1Id = G.getMetadata().getNodeIdForVReg(DstReg);
357378 PBQPRAGraph::NodeId N2Id = G.getMetadata().getNodeIdForVReg(SrcReg);
358 const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed1 =
359 &G.getNodeMetadata(N1Id).getOptionRegs();
360 const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed2 =
361 &G.getNodeMetadata(N2Id).getOptionRegs();
379 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed1 =
380 &G.getNodeMetadata(N1Id).getAllowedRegs();
381 const PBQPRAGraph::NodeMetadata::AllowedRegVector *Allowed2 =
382 &G.getNodeMetadata(N2Id).getAllowedRegs();
362383
363384 PBQPRAGraph::EdgeId EId = G.findEdge(N1Id, N2Id);
364385 if (EId == G.invalidEdgeId()) {
383404 private:
384405
385406 void addVirtRegCoalesce(
386 PBQPRAGraph::RawMatrix &CostMat,
387 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed1,
388 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed2,
389 PBQP::PBQPNum Benefit) {
407 PBQPRAGraph::RawMatrix &CostMat,
408 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed1,
409 const PBQPRAGraph::NodeMetadata::AllowedRegVector &Allowed2,
410 PBQP::PBQPNum Benefit) {
390411 assert(CostMat.getRows() == Allowed1.size() + 1 && "Size mismatch.");
391412 assert(CostMat.getCols() == Allowed2.size() + 1 && "Size mismatch.");
392413 for (unsigned I = 0; I != Allowed1.size(); ++I) {
500521 PBQPRAGraph::RawVector NodeCosts(VRegAllowed.size() + 1, 0);
501522 PBQPRAGraph::NodeId NId = G.addNode(std::move(NodeCosts));
502523 G.getNodeMetadata(NId).setVReg(VReg);
503 G.getNodeMetadata(NId).setOptionRegs(std::move(VRegAllowed));
524 G.getNodeMetadata(NId).setAllowedRegs(
525 G.getMetadata().getAllowedRegs(std::move(VRegAllowed)));
504526 G.getMetadata().setNodeIdForVReg(VReg, NId);
505527 }
506528 }
528550 unsigned AllocOption = Solution.getSelection(NId);
529551
530552 if (AllocOption != PBQP::RegAlloc::getSpillOptionIdx()) {
531 unsigned PReg = G.getNodeMetadata(NId).getOptionRegs()[AllocOption - 1];
553 unsigned PReg = G.getNodeMetadata(NId).getAllowedRegs()[AllocOption - 1];
532554 DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> "
533555 << TRI.getName(PReg) << "\n");
534556 assert(PReg != 0 && "Invalid preg selected.");
561583
562584 return !AnotherRoundNeeded;
563585 }
564
565586
566587 void RegAllocPBQP::finalizeAlloc(MachineFunction &MF,
567588 LiveIntervals &LIS,
173173 PBQPRAGraph::NodeId node1 = G.getMetadata().getNodeIdForVReg(Rd);
174174 PBQPRAGraph::NodeId node2 = G.getMetadata().getNodeIdForVReg(Ra);
175175
176 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRdAllowed =
177 &G.getNodeMetadata(node1).getOptionRegs();
178 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRaAllowed =
179 &G.getNodeMetadata(node2).getOptionRegs();
176 const PBQPRAGraph::NodeMetadata::AllowedRegVector *vRdAllowed =
177 &G.getNodeMetadata(node1).getAllowedRegs();
178 const PBQPRAGraph::NodeMetadata::AllowedRegVector *vRaAllowed =
179 &G.getNodeMetadata(node2).getAllowedRegs();
180180
181181 PBQPRAGraph::EdgeId edge = G.findEdge(node1, node2);
182182
267267
268268 const LiveInterval &lr = LIs.getInterval(r);
269269 if (ld.overlaps(lr)) {
270 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRdAllowed =
271 &G.getNodeMetadata(node1).getOptionRegs();
270 const PBQPRAGraph::NodeMetadata::AllowedRegVector *vRdAllowed =
271 &G.getNodeMetadata(node1).getAllowedRegs();
272272
273273 PBQPRAGraph::NodeId node2 = G.getMetadata().getNodeIdForVReg(r);
274 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRrAllowed =
275 &G.getNodeMetadata(node2).getOptionRegs();
274 const PBQPRAGraph::NodeMetadata::AllowedRegVector *vRrAllowed =
275 &G.getNodeMetadata(node2).getAllowedRegs();
276276
277277 PBQPRAGraph::EdgeId edge = G.findEdge(node1, node2);
278278 assert(edge != G.invalidEdgeId() &&