llvm.org GIT mirror llvm / 54d63b4
[PBQP] Replace PBQPBuilder with composable constraints (PBQPRAConstraint). This patch removes the PBQPBuilder class and its subclasses and replaces them with a composable constraints class: PBQPRAConstraint. This allows constraints that are only required for optimisation (e.g. coalescing, soft pairing) to be mixed and matched. This patch also introduces support for target writers to supply custom constraints for their targets by overriding a TargetSubtargetInfo method: std::unique_ptr<PBQPRAConstraints> getCustomPBQPConstraints() const; This patch should have no effect on allocations. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219421 91177308-0d34-0410-b5e6-96231b3b80d8 Lang Hames 5 years ago
14 changed file(s) with 557 addition(s) and 684 deletion(s). Raw diff Collapse all Expand all
2121 #include
2222 #include
2323
24 namespace llvm {
2425 namespace PBQP {
2526
2627 template
103104 MatrixCostPool matrixPool;
104105 };
105106
106 }
107 } // namespace PBQP
108 } // namespace llvm
107109
108110 #endif
1616
1717 #include "llvm/ADT/ilist.h"
1818 #include "llvm/ADT/ilist_node.h"
19 #include "llvm/Support/Compiler.h"
19 #include "llvm/Support/Debug.h"
2020 #include
2121 #include
2222 #include
2323
24 namespace llvm {
2425 namespace PBQP {
2526
2627 class GraphBase {
194195 EdgeEntry& getEdge(EdgeId EId) { return Edges[EId]; }
195196 const EdgeEntry& getEdge(EdgeId EId) const { return Edges[EId]; }
196197
197 NodeId addConstructedNode(const NodeEntry &N) {
198 NodeId addConstructedNode(NodeEntry N) {
198199 NodeId NId = 0;
199200 if (!FreeNodeIds.empty()) {
200201 NId = FreeNodeIds.back();
207208 return NId;
208209 }
209210
210 EdgeId addConstructedEdge(const EdgeEntry &E) {
211 EdgeId addConstructedEdge(EdgeEntry E) {
211212 assert(findEdge(E.getN1Id(), E.getN2Id()) == invalidEdgeId() &&
212213 "Attempt to add duplicate edge.");
213214 EdgeId EId = 0;
236237
237238 class NodeItr {
238239 public:
240 typedef std::forward_iterator_tag iterator_category;
241 typedef NodeId value_type;
242 typedef int difference_type;
243 typedef NodeId* pointer;
244 typedef NodeId& reference;
245
239246 NodeItr(NodeId CurNId, const Graph &G)
240247 : CurNId(CurNId), EndNId(G.Nodes.size()), FreeNodeIds(G.FreeNodeIds) {
241248 this->CurNId = findNextInUse(CurNId); // Move to first in-use node id
250257 NodeId findNextInUse(NodeId NId) const {
251258 while (NId < EndNId &&
252259 std::find(FreeNodeIds.begin(), FreeNodeIds.end(), NId) !=
253 FreeNodeIds.end()) {
260 FreeNodeIds.end()) {
254261 ++NId;
255262 }
256263 return NId;
330337 };
331338
332339 /// @brief Construct an empty PBQP graph.
333 Graph() : Solver(nullptr) { }
340 Graph() : Solver(nullptr) {}
341
342 /// @brief Construct an empty PBQP graph with the given graph metadata.
343 Graph(GraphMetadata Metadata) : Metadata(Metadata), Solver(nullptr) {}
334344
335345 /// @brief Get a reference to the graph metadata.
336346 GraphMetadata& getMetadata() { return Metadata; }
417427 /// @brief Get a node's cost vector (const version).
418428 /// @param NId Node id.
419429 /// @return Node cost vector.
420 const Vector& getNodeCosts(NodeId NId) const {
421 return *getNode(NId).Costs;
422 }
430 const Vector& getNodeCosts(NodeId NId) const { return *getNode(NId).Costs; }
423431
424432 NodeMetadata& getNodeMetadata(NodeId NId) {
425433 return getNode(NId).Metadata;
447455 /// @brief Get an edge's cost matrix (const version).
448456 /// @param EId Edge id.
449457 /// @return Edge cost matrix.
450 const Matrix& getEdgeCosts(EdgeId EId) const { return *getEdge(EId).Costs; }
458 const Matrix& getEdgeCosts(EdgeId EId) const {
459 return *getEdge(EId).Costs;
460 }
451461
452462 EdgeMetadata& getEdgeMetadata(EdgeId NId) {
453463 return getEdge(NId).Metadata;
506516 NodeEntry &N = getNode(NId);
507517 // TODO: Can this be for-each'd?
508518 for (AdjEdgeItr AEItr = N.adjEdgesBegin(),
509 AEEnd = N.adjEdgesEnd();
519 AEEnd = N.adjEdgesEnd();
510520 AEItr != AEEnd;) {
511521 EdgeId EId = *AEItr;
512522 ++AEItr;
587597
588598 /// @brief Dump a graph to an output stream.
589599 template
590 void dump(OStream &OS) {
600 void dumpToStream(OStream &OS) {
591601 OS << nodeIds().size() << " " << edgeIds().size() << "\n";
592602
593603 for (auto NId : nodeIds()) {
620630 }
621631 }
622632
633 /// @brief Dump this graph to dbgs().
634 void dump() {
635 dumpToStream(dbgs());
636 }
637
623638 /// @brief Print a representation of this graph in DOT format.
624639 /// @param OS Output stream to print on.
625640 template
644659 }
645660 };
646661
647 }
662 } // namespace PBQP
663 } // namespace llvm
648664
649665 #endif // LLVM_CODEGEN_PBQP_GRAPH_HPP
1313 #include
1414 #include
1515
16 namespace llvm {
1617 namespace PBQP {
1718
1819 typedef float PBQPNum;
432433 Metadata md;
433434 };
434435
435 }
436 } // namespace PBQP
437 } // namespace llvm
436438
437439 #endif // LLVM_CODEGEN_PBQP_MATH_H
1717 #include "Math.h"
1818 #include "Solution.h"
1919
20 namespace llvm {
2021 namespace PBQP {
2122
2223 /// \brief Reduce a node of degree one.
185186 return s;
186187 }
187188
188 }
189 } // namespace PBQP
190 } // namespace llvm
189191
190192 #endif
2525 #include
2626 #include
2727
28 namespace llvm{
2829 namespace PBQP {
29
3030 namespace RegAlloc {
31
32 /// @brief Spill option index.
33 inline unsigned getSpillOptionIdx() { return 0; }
3134
3235 /// \brief Metadata to speed allocatability test.
3336 ///
8083
8184 class NodeMetadata {
8285 public:
86 typedef std::vector OptionToRegMap;
87
8388 typedef enum { Unprocessed,
8489 OptimallyReducible,
8590 ConservativelyAllocatable,
8792
8893 NodeMetadata() : RS(Unprocessed), DeniedOpts(0), OptUnsafeEdges(nullptr){}
8994 ~NodeMetadata() { delete[] OptUnsafeEdges; }
95
96 void setVReg(unsigned VReg) { this->VReg = VReg; }
97 unsigned getVReg() const { return VReg; }
98
99 void setOptionRegs(OptionToRegMap OptionRegs) {
100 this->OptionRegs = std::move(OptionRegs);
101 }
102 const OptionToRegMap& getOptionRegs() const { return OptionRegs; }
90103
91104 void setup(const Vector& Costs) {
92105 NumOpts = Costs.getLength() - 1;
123136 unsigned NumOpts;
124137 unsigned DeniedOpts;
125138 unsigned* OptUnsafeEdges;
139 unsigned VReg;
140 OptionToRegMap OptionRegs;
126141 };
127142
128143 class RegAllocSolverImpl {
143158 typedef RegAlloc::NodeMetadata NodeMetadata;
144159
145160 struct EdgeMetadata { };
146 struct GraphMetadata { };
161
162 class GraphMetadata {
163 public:
164 GraphMetadata(MachineFunction &MF,
165 LiveIntervals &LIS,
166 MachineBlockFrequencyInfo &MBFI)
167 : MF(MF), LIS(LIS), MBFI(MBFI) {}
168
169 MachineFunction &MF;
170 LiveIntervals &LIS;
171 MachineBlockFrequencyInfo &MBFI;
172
173 void setNodeIdForVReg(unsigned VReg, GraphBase::NodeId NId) {
174 VRegToNodeId[VReg] = NId;
175 }
176
177 GraphBase::NodeId getNodeIdForVReg(unsigned VReg) const {
178 auto VRegItr = VRegToNodeId.find(VReg);
179 if (VRegItr == VRegToNodeId.end())
180 return GraphBase::invalidNodeId();
181 return VRegItr->second;
182 }
183
184 void eraseNodeIdForVReg(unsigned VReg) {
185 VRegToNodeId.erase(VReg);
186 }
187
188 private:
189 DenseMap VRegToNodeId;
190 };
147191
148192 typedef PBQP::Graph Graph;
149193
344388 NodeSet NotProvablyAllocatableNodes;
345389 };
346390
347 typedef Graph Graph;
348
349 inline Solution solve(Graph& G) {
391 class PBQPRAGraph : public PBQP::Graph {
392 private:
393 typedef PBQP::Graph BaseT;
394 public:
395 PBQPRAGraph(GraphMetadata Metadata) : BaseT(Metadata) {}
396 };
397
398 inline Solution solve(PBQPRAGraph& G) {
350399 if (G.empty())
351400 return Solution();
352401 RegAllocSolverImpl RegAllocSolver(G);
353402 return RegAllocSolver.solve();
354403 }
355
356 }
357 }
404 } // namespace RegAlloc
405 } // namespace PBQP
406 } // namespace llvm
358407
359408 #endif // LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H
1717 #include "Math.h"
1818 #include
1919
20 namespace llvm {
2021 namespace PBQP {
2122
2223 /// \brief Represents a solution to a PBQP problem.
8687
8788 };
8889
89 }
90 } // namespace PBQP
91 } // namespace llvm
9092
9193 #endif // LLVM_CODEGEN_PBQP_SOLUTION_H
1515 #ifndef LLVM_CODEGEN_REGALLOCPBQP_H
1616 #define LLVM_CODEGEN_REGALLOCPBQP_H
1717
18 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/SmallVector.h"
2018 #include "llvm/CodeGen/MachineFunctionPass.h"
19 #include "llvm/CodeGen/PBQPRAConstraint.h"
2120 #include "llvm/CodeGen/PBQP/RegAllocSolver.h"
22 #include
23 #include
2421
2522 namespace llvm {
2623
27 class LiveIntervals;
28 class MachineBlockFrequencyInfo;
29 class MachineFunction;
30 class TargetRegisterInfo;
31
32 typedef PBQP::RegAlloc::Graph PBQPRAGraph;
33
34 /// This class wraps up a PBQP instance representing a register allocation
35 /// problem, plus the structures necessary to map back from the PBQP solution
36 /// to a register allocation solution. (i.e. The PBQP-node <--> vreg map,
37 /// and the PBQP option <--> storage location map).
38 class PBQPRAProblem {
39 public:
40
41 typedef SmallVector AllowedSet;
42
43 PBQPRAGraph& getGraph() { return graph; }
44
45 const PBQPRAGraph& getGraph() const { return graph; }
46
47 /// Record the mapping between the given virtual register and PBQP node,
48 /// and the set of allowed pregs for the vreg.
49 ///
50 /// If you are extending
51 /// PBQPBuilder you are unlikely to need this: Nodes and options for all
52 /// vregs will already have been set up for you by the base class.
53 template
54 void recordVReg(unsigned vreg, PBQPRAGraph::NodeId nodeId,
55 AllowedRegsItr arBegin, AllowedRegsItr arEnd) {
56 assert(node2VReg.find(nodeId) == node2VReg.end() && "Re-mapping node.");
57 assert(vreg2Node.find(vreg) == vreg2Node.end() && "Re-mapping vreg.");
58 assert(allowedSets[vreg].empty() && "vreg already has pregs.");
59
60 node2VReg[nodeId] = vreg;
61 vreg2Node[vreg] = nodeId;
62 std::copy(arBegin, arEnd, std::back_inserter(allowedSets[vreg]));
63 }
64
65 /// Get the virtual register corresponding to the given PBQP node.
66 unsigned getVRegForNode(PBQPRAGraph::NodeId nodeId) const;
67
68 /// Get the PBQP node corresponding to the given virtual register.
69 PBQPRAGraph::NodeId getNodeForVReg(unsigned vreg) const;
70
71 /// Returns true if the given PBQP option represents a physical register,
72 /// false otherwise.
73 bool isPRegOption(unsigned vreg, unsigned option) const {
74 // At present we only have spills or pregs, so anything that's not a
75 // spill is a preg. (This might be extended one day to support remat).
76 return !isSpillOption(vreg, option);
77 }
78
79 /// Returns true if the given PBQP option represents spilling, false
80 /// otherwise.
81 bool isSpillOption(unsigned vreg, unsigned option) const {
82 // We hardcode option zero as the spill option.
83 return option == 0;
84 }
85
86 /// Returns the allowed set for the given virtual register.
87 const AllowedSet& getAllowedSet(unsigned vreg) const;
88
89 /// Get PReg for option.
90 unsigned getPRegForOption(unsigned vreg, unsigned option) const;
91
92 private:
93
94 typedef std::map Node2VReg;
95 typedef DenseMap VReg2Node;
96 typedef DenseMap AllowedSetMap;
97
98 PBQPRAGraph graph;
99 Node2VReg node2VReg;
100 VReg2Node vreg2Node;
101
102 AllowedSetMap allowedSets;
103
104 };
105
106 /// Builds PBQP instances to represent register allocation problems. Includes
107 /// spill, interference and coalescing costs by default. You can extend this
108 /// class to support additional constraints for your architecture.
109 class PBQPBuilder {
110 private:
111 PBQPBuilder(const PBQPBuilder&) LLVM_DELETED_FUNCTION;
112 void operator=(const PBQPBuilder&) LLVM_DELETED_FUNCTION;
113 public:
114
115 typedef std::set RegSet;
116
117 /// Default constructor.
118 PBQPBuilder() {}
119
120 /// Clean up a PBQPBuilder.
121 virtual ~PBQPBuilder() {}
122
123 /// Build a PBQP instance to represent the register allocation problem for
124 /// the given MachineFunction.
125 virtual std::unique_ptr
126 build(MachineFunction *mf, const LiveIntervals *lis,
127 const MachineBlockFrequencyInfo *mbfi, const RegSet &vregs);
128
129 private:
130
131 void addSpillCosts(PBQP::Vector &costVec, PBQP::PBQPNum spillCost);
132
133 void addInterferenceCosts(PBQP::Matrix &costMat,
134 const PBQPRAProblem::AllowedSet &vr1Allowed,
135 const PBQPRAProblem::AllowedSet &vr2Allowed,
136 const TargetRegisterInfo *tri);
137 };
138
139 /// Extended builder which adds coalescing constraints to a problem.
140 class PBQPBuilderWithCoalescing : public PBQPBuilder {
141 public:
142
143 /// Build a PBQP instance to represent the register allocation problem for
144 /// the given MachineFunction.
145 std::unique_ptr build(MachineFunction *mf,
146 const LiveIntervals *lis,
147 const MachineBlockFrequencyInfo *mbfi,
148 const RegSet &vregs) override;
149
150 private:
151
152 void addPhysRegCoalesce(PBQP::Vector &costVec, unsigned pregOption,
153 PBQP::PBQPNum benefit);
154
155 void addVirtRegCoalesce(PBQP::Matrix &costMat,
156 const PBQPRAProblem::AllowedSet &vr1Allowed,
157 const PBQPRAProblem::AllowedSet &vr2Allowed,
158 PBQP::PBQPNum benefit);
159 };
160
24 /// @brief Create a PBQP register allocator instance.
16125 FunctionPass *
162 createPBQPRegisterAllocator(std::unique_ptr builder,
163 char *customPassID = nullptr);
26 createPBQPRegisterAllocator(char *customPassID = nullptr);
16427 }
16528
16629 #endif /* LLVM_CODEGEN_REGALLOCPBQP_H */
1313 #ifndef LLVM_TARGET_TARGETSUBTARGETINFO_H
1414 #define LLVM_TARGET_TARGETSUBTARGETINFO_H
1515
16 #include "llvm/CodeGen/PBQPRAConstraint.h"
1617 #include "llvm/MC/MCSubtargetInfo.h"
1718 #include "llvm/Support/CodeGen.h"
1819
159160
160161 /// \brief Enable the use of the early if conversion pass.
161162 virtual bool enableEarlyIfConversion() const { return false; }
163
164 /// \brief Return PBQPConstraint(s) for the target.
165 ///
166 /// Override to provide custom PBQP constraints.
167 virtual std::unique_ptr getCustomPBQPConstraints() const {
168 return nullptr;
169 }
162170 };
163171
164172 } // End llvm namespace
6161 #define DEBUG_TYPE "regalloc"
6262
6363 static RegisterRegAlloc
64 registerPBQPRepAlloc("pbqp", "PBQP register allocator",
64 RegisterPBQPRepAlloc("pbqp", "PBQP register allocator",
6565 createDefaultPBQPRegisterAllocator);
6666
6767 static cl::opt
68 pbqpCoalescing("pbqp-coalescing",
68 PBQPCoalescing("pbqp-coalescing",
6969 cl::desc("Attempt coalescing during PBQP register allocation."),
7070 cl::init(false), cl::Hidden);
7171
7272 #ifndef NDEBUG
7373 static cl::opt
74 pbqpDumpGraphs("pbqp-dump-graphs",
74 PBQPDumpGraphs("pbqp-dump-graphs",
7575 cl::desc("Dump graphs for each function/round in the compilation unit."),
7676 cl::init(false), cl::Hidden);
7777 #endif
8888 static char ID;
8989
9090 /// Construct a PBQP register allocator.
91 RegAllocPBQP(std::unique_ptr b, char *cPassID = nullptr)
92 : MachineFunctionPass(ID), builder(std::move(b)), customPassID(cPassID) {
91 RegAllocPBQP(char *cPassID = nullptr)
92 : MachineFunctionPass(ID), customPassID(cPassID) {
9393 initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
9494 initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
9595 initializeLiveStacksPass(*PassRegistry::getPassRegistry());
117117 typedef std::map CoalesceMap;
118118 typedef std::set RegSet;
119119
120 std::unique_ptr builder;
121
122120 char *customPassID;
123121
124 MachineFunction *mf;
125 const TargetMachine *tm;
126 const TargetRegisterInfo *tri;
127 const TargetInstrInfo *tii;
128 MachineRegisterInfo *mri;
129 const MachineBlockFrequencyInfo *mbfi;
130
131 std::unique_ptr spiller;
132 LiveIntervals *lis;
133 LiveStacks *lss;
134 VirtRegMap *vrm;
135
136 RegSet vregsToAlloc, emptyIntervalVRegs;
122 RegSet VRegsToAlloc, EmptyIntervalVRegs;
137123
138124 /// \brief Finds the initial set of vreg intervals to allocate.
139 void findVRegIntervalsToAlloc();
125 void findVRegIntervalsToAlloc(const MachineFunction &MF, LiveIntervals &LIS);
126
127 /// \brief Constructs an initial graph.
128 void initializeGraph(PBQPRAGraph &G);
140129
141130 /// \brief Given a solved PBQP problem maps this solution back to a register
142131 /// assignment.
143 bool mapPBQPToRegAlloc(const PBQPRAProblem &problem,
144 const PBQP::Solution &solution);
132 bool mapPBQPToRegAlloc(const PBQPRAGraph &G,
133 const PBQP::Solution &Solution,
134 VirtRegMap &VRM,
135 Spiller &VRegSpiller);
145136
146137 /// \brief Postprocessing before final spilling. Sets basic block "live in"
147138 /// variables.
148 void finalizeAlloc() const;
139 void finalizeAlloc(MachineFunction &MF, LiveIntervals &LIS,
140 VirtRegMap &VRM) const;
149141
150142 };
151143
152144 char RegAllocPBQP::ID = 0;
153145
146 /// @brief Set spill costs for each node in the PBQP reg-alloc graph.
147 class SpillCosts : public PBQPRAConstraint {
148 public:
149 void apply(PBQPRAGraph &G) override {
150 LiveIntervals &LIS = G.getMetadata().LIS;
151
152 for (auto NId : G.nodeIds()) {
153 PBQP::PBQPNum SpillCost =
154 LIS.getInterval(G.getNodeMetadata(NId).getVReg()).weight;
155 if (SpillCost == 0.0)
156 SpillCost = std::numeric_limits::min();
157 PBQPRAGraph::RawVector NodeCosts(G.getNodeCosts(NId));
158 NodeCosts[PBQP::RegAlloc::getSpillOptionIdx()] = SpillCost;
159 G.setNodeCosts(NId, std::move(NodeCosts));
160 }
161 }
162 };
163
164 /// @brief Add interference edges between overlapping vregs.
165 class Interference : public PBQPRAConstraint {
166 public:
167
168 void apply(PBQPRAGraph &G) override {
169 LiveIntervals &LIS = G.getMetadata().LIS;
170 const TargetRegisterInfo &TRI =
171 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
172
173 for (auto NItr = G.nodeIds().begin(), NEnd = G.nodeIds().end();
174 NItr != NEnd; ++NItr) {
175 auto NId = *NItr;
176 unsigned NVReg = G.getNodeMetadata(NId).getVReg();
177 LiveInterval &NLI = LIS.getInterval(NVReg);
178
179 for (auto MItr = std::next(NItr); MItr != NEnd; ++MItr) {
180 auto MId = *MItr;
181 unsigned MVReg = G.getNodeMetadata(MId).getVReg();
182 LiveInterval &MLI = LIS.getInterval(MVReg);
183
184 if (NLI.overlaps(MLI)) {
185 const auto &NOpts = G.getNodeMetadata(NId).getOptionRegs();
186 const auto &MOpts = G.getNodeMetadata(MId).getOptionRegs();
187 G.addEdge(NId, MId, createInterferenceMatrix(TRI, NOpts, MOpts));
188 }
189 }
190 }
191 }
192
193 private:
194
195 PBQPRAGraph::RawMatrix createInterferenceMatrix(
196 const TargetRegisterInfo &TRI,
197 const PBQPRAGraph::NodeMetadata::OptionToRegMap &NOpts,
198 const PBQPRAGraph::NodeMetadata::OptionToRegMap &MOpts) {
199 PBQPRAGraph::RawMatrix M(NOpts.size() + 1, MOpts.size() + 1, 0);
200 for (unsigned I = 0; I != NOpts.size(); ++I) {
201 unsigned PRegN = NOpts[I];
202 for (unsigned J = 0; J != MOpts.size(); ++J) {
203 unsigned PRegM = MOpts[J];
204 if (TRI.regsOverlap(PRegN, PRegM))
205 M[I + 1][J + 1] = std::numeric_limits::infinity();
206 }
207 }
208
209 return M;
210 }
211 };
212
213
214 class Coalescing : public PBQPRAConstraint {
215 public:
216 void apply(PBQPRAGraph &G) override {
217 MachineFunction &MF = G.getMetadata().MF;
218 MachineBlockFrequencyInfo &MBFI = G.getMetadata().MBFI;
219 CoalescerPair CP(*MF.getTarget().getSubtargetImpl()->getRegisterInfo());
220
221 // Scan the machine function and add a coalescing cost whenever CoalescerPair
222 // gives the Ok.
223 for (const auto &MBB : MF) {
224 for (const auto &MI : MBB) {
225
226 // Skip not-coalescable or already coalesced copies.
227 if (!CP.setRegisters(&MI) || CP.getSrcReg() == CP.getDstReg())
228 continue;
229
230 unsigned DstReg = CP.getDstReg();
231 unsigned SrcReg = CP.getSrcReg();
232
233 const float CopyFactor = 0.5; // Cost of copy relative to load. Current
234 // value plucked randomly out of the air.
235
236 PBQP::PBQPNum CBenefit =
237 CopyFactor * LiveIntervals::getSpillWeight(false, true, &MBFI, &MI);
238
239 if (CP.isPhys()) {
240 if (!MF.getRegInfo().isAllocatable(DstReg))
241 continue;
242
243 PBQPRAGraph::NodeId NId = G.getMetadata().getNodeIdForVReg(SrcReg);
244
245 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed =
246 G.getNodeMetadata(NId).getOptionRegs();
247
248 unsigned PRegOpt = 0;
249 while (PRegOpt < Allowed.size() && Allowed[PRegOpt] != DstReg)
250 ++PRegOpt;
251
252 if (PRegOpt < Allowed.size()) {
253 PBQPRAGraph::RawVector NewCosts(G.getNodeCosts(NId));
254 NewCosts[PRegOpt + 1] += CBenefit;
255 G.setNodeCosts(NId, std::move(NewCosts));
256 }
257 } else {
258 PBQPRAGraph::NodeId N1Id = G.getMetadata().getNodeIdForVReg(DstReg);
259 PBQPRAGraph::NodeId N2Id = G.getMetadata().getNodeIdForVReg(SrcReg);
260 const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed1 =
261 &G.getNodeMetadata(N1Id).getOptionRegs();
262 const PBQPRAGraph::NodeMetadata::OptionToRegMap *Allowed2 =
263 &G.getNodeMetadata(N2Id).getOptionRegs();
264
265 PBQPRAGraph::EdgeId EId = G.findEdge(N1Id, N2Id);
266 if (EId == G.invalidEdgeId()) {
267 PBQPRAGraph::RawMatrix Costs(Allowed1->size() + 1,
268 Allowed2->size() + 1, 0);
269 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
270 G.addEdge(N1Id, N2Id, std::move(Costs));
271 } else {
272 if (G.getEdgeNode1Id(EId) == N2Id) {
273 std::swap(N1Id, N2Id);
274 std::swap(Allowed1, Allowed2);
275 }
276 PBQPRAGraph::RawMatrix Costs(G.getEdgeCosts(EId));
277 addVirtRegCoalesce(Costs, *Allowed1, *Allowed2, CBenefit);
278 G.setEdgeCosts(EId, std::move(Costs));
279 }
280 }
281 }
282 }
283 }
284
285 private:
286
287 void addVirtRegCoalesce(
288 PBQPRAGraph::RawMatrix &CostMat,
289 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed1,
290 const PBQPRAGraph::NodeMetadata::OptionToRegMap &Allowed2,
291 PBQP::PBQPNum Benefit) {
292 assert(CostMat.getRows() == Allowed1.size() + 1 && "Size mismatch.");
293 assert(CostMat.getCols() == Allowed2.size() + 1 && "Size mismatch.");
294 for (unsigned I = 0; I != Allowed1.size(); ++I) {
295 unsigned PReg1 = Allowed1[I];
296 for (unsigned J = 0; J != Allowed2.size(); ++J) {
297 unsigned PReg2 = Allowed2[J];
298 if (PReg1 == PReg2)
299 CostMat[I + 1][J + 1] += -Benefit;
300 }
301 }
302 }
303
304 };
305
154306 } // End anonymous namespace.
155307
156 unsigned PBQPRAProblem::getVRegForNode(PBQPRAGraph::NodeId node) const {
157 Node2VReg::const_iterator vregItr = node2VReg.find(node);
158 assert(vregItr != node2VReg.end() && "No vreg for node.");
159 return vregItr->second;
160 }
161
162 PBQPRAGraph::NodeId PBQPRAProblem::getNodeForVReg(unsigned vreg) const {
163 VReg2Node::const_iterator nodeItr = vreg2Node.find(vreg);
164 assert(nodeItr != vreg2Node.end() && "No node for vreg.");
165 return nodeItr->second;
166
167 }
168
169 const PBQPRAProblem::AllowedSet&
170 PBQPRAProblem::getAllowedSet(unsigned vreg) const {
171 AllowedSetMap::const_iterator allowedSetItr = allowedSets.find(vreg);
172 assert(allowedSetItr != allowedSets.end() && "No pregs for vreg.");
173 const AllowedSet &allowedSet = allowedSetItr->second;
174 return allowedSet;
175 }
176
177 unsigned PBQPRAProblem::getPRegForOption(unsigned vreg, unsigned option) const {
178 assert(isPRegOption(vreg, option) && "Not a preg option.");
179
180 const AllowedSet& allowedSet = getAllowedSet(vreg);
181 assert(option <= allowedSet.size() && "Option outside allowed set.");
182 return allowedSet[option - 1];
183 }
184
185 std::unique_ptr
186 PBQPBuilder::build(MachineFunction *mf, const LiveIntervals *lis,
187 const MachineBlockFrequencyInfo *mbfi, const RegSet &vregs) {
188
189 LiveIntervals *LIS = const_cast(lis);
190 MachineRegisterInfo *mri = &mf->getRegInfo();
191 const TargetRegisterInfo *tri = mf->getSubtarget().getRegisterInfo();
192
193 auto p = llvm::make_unique();
194 PBQPRAGraph &g = p->getGraph();
195 RegSet pregs;
196
197 // Collect the set of preg intervals, record that they're used in the MF.
198 for (unsigned Reg = 1, e = tri->getNumRegs(); Reg != e; ++Reg) {
199 if (mri->def_empty(Reg))
200 continue;
201 pregs.insert(Reg);
202 mri->setPhysRegUsed(Reg);
203 }
204
205 // Iterate over vregs.
206 for (RegSet::const_iterator vregItr = vregs.begin(), vregEnd = vregs.end();
207 vregItr != vregEnd; ++vregItr) {
208 unsigned vreg = *vregItr;
209 const TargetRegisterClass *trc = mri->getRegClass(vreg);
210 LiveInterval *vregLI = &LIS->getInterval(vreg);
211
212 // Record any overlaps with regmask operands.
213 BitVector regMaskOverlaps;
214 LIS->checkRegMaskInterference(*vregLI, regMaskOverlaps);
215
216 // Compute an initial allowed set for the current vreg.
217 typedef std::vector VRAllowed;
218 VRAllowed vrAllowed;
219 ArrayRef rawOrder = trc->getRawAllocationOrder(*mf);
220 for (unsigned i = 0; i != rawOrder.size(); ++i) {
221 unsigned preg = rawOrder[i];
222 if (mri->isReserved(preg))
223 continue;
224
225 // vregLI crosses a regmask operand that clobbers preg.
226 if (!regMaskOverlaps.empty() && !regMaskOverlaps.test(preg))
227 continue;
228
229 // vregLI overlaps fixed regunit interference.
230 bool Interference = false;
231 for (MCRegUnitIterator Units(preg, tri); Units.isValid(); ++Units) {
232 if (vregLI->overlaps(LIS->getRegUnit(*Units))) {
233 Interference = true;
234 break;
235 }
236 }
237 if (Interference)
238 continue;
239
240 // preg is usable for this virtual register.
241 vrAllowed.push_back(preg);
242 }
243
244 PBQP::Vector nodeCosts(vrAllowed.size() + 1, 0);
245
246 PBQP::PBQPNum spillCost = (vregLI->weight != 0.0) ?
247 vregLI->weight : std::numeric_limits::min();
248
249 addSpillCosts(nodeCosts, spillCost);
250
251 // Construct the node.
252 PBQPRAGraph::NodeId nId = g.addNode(std::move(nodeCosts));
253
254 // Record the mapping and allowed set in the problem.
255 p->recordVReg(vreg, nId, vrAllowed.begin(), vrAllowed.end());
256
257 }
258
259 for (RegSet::const_iterator vr1Itr = vregs.begin(), vrEnd = vregs.end();
260 vr1Itr != vrEnd; ++vr1Itr) {
261 unsigned vr1 = *vr1Itr;
262 const LiveInterval &l1 = lis->getInterval(vr1);
263 const PBQPRAProblem::AllowedSet &vr1Allowed = p->getAllowedSet(vr1);
264
265 for (RegSet::const_iterator vr2Itr = std::next(vr1Itr); vr2Itr != vrEnd;
266 ++vr2Itr) {
267 unsigned vr2 = *vr2Itr;
268 const LiveInterval &l2 = lis->getInterval(vr2);
269 const PBQPRAProblem::AllowedSet &vr2Allowed = p->getAllowedSet(vr2);
270
271 assert(!l2.empty() && "Empty interval in vreg set?");
272 if (l1.overlaps(l2)) {
273 PBQP::Matrix edgeCosts(vr1Allowed.size()+1, vr2Allowed.size()+1, 0);
274 addInterferenceCosts(edgeCosts, vr1Allowed, vr2Allowed, tri);
275
276 g.addEdge(p->getNodeForVReg(vr1), p->getNodeForVReg(vr2),
277 std::move(edgeCosts));
278 }
279 }
280 }
281
282 return p;
283 }
284
285 void PBQPBuilder::addSpillCosts(PBQP::Vector &costVec,
286 PBQP::PBQPNum spillCost) {
287 costVec[0] = spillCost;
288 }
289
290 void PBQPBuilder::addInterferenceCosts(
291 PBQP::Matrix &costMat,
292 const PBQPRAProblem::AllowedSet &vr1Allowed,
293 const PBQPRAProblem::AllowedSet &vr2Allowed,
294 const TargetRegisterInfo *tri) {
295 assert(costMat.getRows() == vr1Allowed.size() + 1 && "Matrix height mismatch.");
296 assert(costMat.getCols() == vr2Allowed.size() + 1 && "Matrix width mismatch.");
297
298 for (unsigned i = 0; i != vr1Allowed.size(); ++i) {
299 unsigned preg1 = vr1Allowed[i];
300
301 for (unsigned j = 0; j != vr2Allowed.size(); ++j) {
302 unsigned preg2 = vr2Allowed[j];
303
304 if (tri->regsOverlap(preg1, preg2)) {
305 costMat[i + 1][j + 1] = std::numeric_limits::infinity();
306 }
307 }
308 }
309 }
310
311 std::unique_ptr
312 PBQPBuilderWithCoalescing::build(MachineFunction *mf, const LiveIntervals *lis,
313 const MachineBlockFrequencyInfo *mbfi,
314 const RegSet &vregs) {
315
316 std::unique_ptr p = PBQPBuilder::build(mf, lis, mbfi, vregs);
317 PBQPRAGraph &g = p->getGraph();
318
319 const TargetMachine &tm = mf->getTarget();
320 CoalescerPair cp(*tm.getSubtargetImpl()->getRegisterInfo());
321
322 // Scan the machine function and add a coalescing cost whenever CoalescerPair
323 // gives the Ok.
324 for (const auto &mbb : *mf) {
325 for (const auto &mi : mbb) {
326 if (!cp.setRegisters(&mi)) {
327 continue; // Not coalescable.
328 }
329
330 if (cp.getSrcReg() == cp.getDstReg()) {
331 continue; // Already coalesced.
332 }
333
334 unsigned dst = cp.getDstReg(),
335 src = cp.getSrcReg();
336
337 const float copyFactor = 0.5; // Cost of copy relative to load. Current
338 // value plucked randomly out of the air.
339
340 PBQP::PBQPNum cBenefit =
341 copyFactor * LiveIntervals::getSpillWeight(false, true, mbfi, &mi);
342
343 if (cp.isPhys()) {
344 if (!mf->getRegInfo().isAllocatable(dst)) {
345 continue;
346 }
347
348 const PBQPRAProblem::AllowedSet &allowed = p->getAllowedSet(src);
349 unsigned pregOpt = 0;
350 while (pregOpt < allowed.size() && allowed[pregOpt] != dst) {
351 ++pregOpt;
352 }
353 if (pregOpt < allowed.size()) {
354 ++pregOpt; // +1 to account for spill option.
355 PBQPRAGraph::NodeId node = p->getNodeForVReg(src);
356 DEBUG(llvm::dbgs() << "Reading node costs for node " << node << "\n");
357 DEBUG(llvm::dbgs() << "Source node: " << &g.getNodeCosts(node) << "\n");
358 PBQP::Vector newCosts(g.getNodeCosts(node));
359 addPhysRegCoalesce(newCosts, pregOpt, cBenefit);
360 g.setNodeCosts(node, newCosts);
361 }
362 } else {
363 const PBQPRAProblem::AllowedSet *allowed1 = &p->getAllowedSet(dst);
364 const PBQPRAProblem::AllowedSet *allowed2 = &p->getAllowedSet(src);
365 PBQPRAGraph::NodeId node1 = p->getNodeForVReg(dst);
366 PBQPRAGraph::NodeId node2 = p->getNodeForVReg(src);
367 PBQPRAGraph::EdgeId edge = g.findEdge(node1, node2);
368 if (edge == g.invalidEdgeId()) {
369 PBQP::Matrix costs(allowed1->size() + 1, allowed2->size() + 1, 0);
370 addVirtRegCoalesce(costs, *allowed1, *allowed2, cBenefit);
371 g.addEdge(node1, node2, costs);
372 } else {
373 if (g.getEdgeNode1Id(edge) == node2) {
374 std::swap(node1, node2);
375 std::swap(allowed1, allowed2);
376 }
377 PBQP::Matrix costs(g.getEdgeCosts(edge));
378 addVirtRegCoalesce(costs, *allowed1, *allowed2, cBenefit);
379 g.setEdgeCosts(edge, costs);
380 }
381 }
382 }
383 }
384
385 return p;
386 }
387
388 void PBQPBuilderWithCoalescing::addPhysRegCoalesce(PBQP::Vector &costVec,
389 unsigned pregOption,
390 PBQP::PBQPNum benefit) {
391 costVec[pregOption] += -benefit;
392 }
393
394 void PBQPBuilderWithCoalescing::addVirtRegCoalesce(
395 PBQP::Matrix &costMat,
396 const PBQPRAProblem::AllowedSet &vr1Allowed,
397 const PBQPRAProblem::AllowedSet &vr2Allowed,
398 PBQP::PBQPNum benefit) {
399
400 assert(costMat.getRows() == vr1Allowed.size() + 1 && "Size mismatch.");
401 assert(costMat.getCols() == vr2Allowed.size() + 1 && "Size mismatch.");
402
403 for (unsigned i = 0; i != vr1Allowed.size(); ++i) {
404 unsigned preg1 = vr1Allowed[i];
405 for (unsigned j = 0; j != vr2Allowed.size(); ++j) {
406 unsigned preg2 = vr2Allowed[j];
407
408 if (preg1 == preg2) {
409 costMat[i + 1][j + 1] += -benefit;
410 }
411 }
412 }
413 }
414
308 // Out-of-line destructor/anchor for PBQPRAConstraint.
309 PBQPRAConstraint::~PBQPRAConstraint() {}
310 void PBQPRAConstraint::anchor() {}
311 void PBQPRAConstraintList::anchor() {}
415312
416313 void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const {
417314 au.setPreservesCFG();
437334 MachineFunctionPass::getAnalysisUsage(au);
438335 }
439336
440 void RegAllocPBQP::findVRegIntervalsToAlloc() {
337 void RegAllocPBQP::findVRegIntervalsToAlloc(const MachineFunction &MF,
338 LiveIntervals &LIS) {
339 const MachineRegisterInfo &MRI = MF.getRegInfo();
441340
442341 // Iterate over all live ranges.
443 for (unsigned i = 0, e = mri->getNumVirtRegs(); i != e; ++i) {
444 unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
445 if (mri->reg_nodbg_empty(Reg))
342 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
343 unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
344 if (MRI.reg_nodbg_empty(Reg))
446345 continue;
447 LiveInterval *li = &lis->getInterval(Reg);
346 LiveInterval &LI = LIS.getInterval(Reg);
448347
449348 // If this live interval is non-empty we will use pbqp to allocate it.
450349 // Empty intervals we allocate in a simple post-processing stage in
451350 // finalizeAlloc.
452 if (!li->empty()) {
453 vregsToAlloc.insert(li->reg);
351 if (!LI.empty()) {
352 VRegsToAlloc.insert(LI.reg);
454353 } else {
455 emptyIntervalVRegs.insert(li->reg);
456 }
457 }
458 }
459
460 bool RegAllocPBQP::mapPBQPToRegAlloc(const PBQPRAProblem &problem,
461 const PBQP::Solution &solution) {
354 EmptyIntervalVRegs.insert(LI.reg);
355 }
356 }
357 }
358
359 void RegAllocPBQP::initializeGraph(PBQPRAGraph &G) {
360 MachineFunction &MF = G.getMetadata().MF;
361
362 LiveIntervals &LIS = G.getMetadata().LIS;
363 const MachineRegisterInfo &MRI = G.getMetadata().MF.getRegInfo();
364 const TargetRegisterInfo &TRI =
365 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
366
367 for (auto VReg : VRegsToAlloc) {
368 const TargetRegisterClass *TRC = MRI.getRegClass(VReg);
369 LiveInterval &VRegLI = LIS.getInterval(VReg);
370
371 // Record any overlaps with regmask operands.
372 BitVector RegMaskOverlaps;
373 LIS.checkRegMaskInterference(VRegLI, RegMaskOverlaps);
374
375 // Compute an initial allowed set for the current vreg.
376 std::vector VRegAllowed;
377 ArrayRef RawPRegOrder = TRC->getRawAllocationOrder(MF);
378 for (unsigned I = 0; I != RawPRegOrder.size(); ++I) {
379 unsigned PReg = RawPRegOrder[I];
380 if (MRI.isReserved(PReg))
381 continue;
382
383 // vregLI crosses a regmask operand that clobbers preg.
384 if (!RegMaskOverlaps.empty() && !RegMaskOverlaps.test(PReg))
385 continue;
386
387 // vregLI overlaps fixed regunit interference.
388 bool Interference = false;
389 for (MCRegUnitIterator Units(PReg, &TRI); Units.isValid(); ++Units) {
390 if (VRegLI.overlaps(LIS.getRegUnit(*Units))) {
391 Interference = true;
392 break;
393 }
394 }
395 if (Interference)
396 continue;
397
398 // preg is usable for this virtual register.
399 VRegAllowed.push_back(PReg);
400 }
401
402 PBQPRAGraph::RawVector NodeCosts(VRegAllowed.size() + 1, 0);
403 PBQPRAGraph::NodeId NId = G.addNode(std::move(NodeCosts));
404 G.getNodeMetadata(NId).setVReg(VReg);
405 G.getNodeMetadata(NId).setOptionRegs(std::move(VRegAllowed));
406 G.getMetadata().setNodeIdForVReg(VReg, NId);
407 }
408 }
409
410 bool RegAllocPBQP::mapPBQPToRegAlloc(const PBQPRAGraph &G,
411 const PBQP::Solution &Solution,
412 VirtRegMap &VRM,
413 Spiller &VRegSpiller) {
414 MachineFunction &MF = G.getMetadata().MF;
415 LiveIntervals &LIS = G.getMetadata().LIS;
416 const TargetRegisterInfo &TRI =
417 *MF.getTarget().getSubtargetImpl()->getRegisterInfo();
418 (void)TRI;
419
462420 // Set to true if we have any spills
463 bool anotherRoundNeeded = false;
421 bool AnotherRoundNeeded = false;
464422
465423 // Clear the existing allocation.
466 vrm->clearAllVirt();
467
468 const PBQPRAGraph &g = problem.getGraph();
424 VRM.clearAllVirt();
425
469426 // Iterate over the nodes mapping the PBQP solution to a register
470427 // assignment.
471 for (auto NId : g.nodeIds()) {
472 unsigned vreg = problem.getVRegForNode(NId);
473 unsigned alloc = solution.getSelection(NId);
474
475 if (problem.isPRegOption(vreg, alloc)) {
476 unsigned preg = problem.getPRegForOption(vreg, alloc);
477 DEBUG(dbgs() << "VREG " << PrintReg(vreg, tri) << " -> "
478 << tri->getName(preg) << "\n");
479 assert(preg != 0 && "Invalid preg selected.");
480 vrm->assignVirt2Phys(vreg, preg);
481 } else if (problem.isSpillOption(vreg, alloc)) {
482 vregsToAlloc.erase(vreg);
483 SmallVector newSpills;
484 LiveRangeEdit LRE(&lis->getInterval(vreg), newSpills, *mf, *lis, vrm);
485 spiller->spill(LRE);
486
487 DEBUG(dbgs() << "VREG " << PrintReg(vreg, tri) << " -> SPILLED (Cost: "
428 for (auto NId : G.nodeIds()) {
429 unsigned VReg = G.getNodeMetadata(NId).getVReg();
430 unsigned AllocOption = Solution.getSelection(NId);
431
432 if (AllocOption != PBQP::RegAlloc::getSpillOptionIdx()) {
433 unsigned PReg = G.getNodeMetadata(NId).getOptionRegs()[AllocOption - 1];
434 DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> "
435 << TRI.getName(PReg) << "\n");
436 assert(PReg != 0 && "Invalid preg selected.");
437 VRM.assignVirt2Phys(VReg, PReg);
438 } else {
439 VRegsToAlloc.erase(VReg);
440 SmallVector NewSpills;
441 LiveRangeEdit LRE(&LIS.getInterval(VReg), NewSpills, MF, LIS, &VRM);
442 VRegSpiller.spill(LRE);
443
444 DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> SPILLED (Cost: "
488445 << LRE.getParent().weight << ", New vregs: ");
489446
490447 // Copy any newly inserted live intervals into the list of regs to
491448 // allocate.
492 for (LiveRangeEdit::iterator itr = LRE.begin(), end = LRE.end();
493 itr != end; ++itr) {
494 LiveInterval &li = lis->getInterval(*itr);
495 assert(!li.empty() && "Empty spill range.");
496 DEBUG(dbgs() << PrintReg(li.reg, tri) << " ");
497 vregsToAlloc.insert(li.reg);
449 for (LiveRangeEdit::iterator I = LRE.begin(), E = LRE.end();
450 I != E; ++I) {
451 LiveInterval &LI = LIS.getInterval(*I);
452 assert(!LI.empty() && "Empty spill range.");
453 DEBUG(dbgs() << PrintReg(LI.reg, &TRI) << " ");
454 VRegsToAlloc.insert(LI.reg);
498455 }
499456
500457 DEBUG(dbgs() << ")\n");
501458
502459 // We need another round if spill intervals were added.
503 anotherRoundNeeded |= !LRE.empty();
504 } else {
505 llvm_unreachable("Unknown allocation option.");
506 }
507 }
508
509 return !anotherRoundNeeded;
510 }
511
512
513 void RegAllocPBQP::finalizeAlloc() const {
460 AnotherRoundNeeded |= !LRE.empty();
461 }
462 }
463
464 return !AnotherRoundNeeded;
465 }
466
467
468 void RegAllocPBQP::finalizeAlloc(MachineFunction &MF,
469 LiveIntervals &LIS,
470 VirtRegMap &VRM) const {
471 MachineRegisterInfo &MRI = MF.getRegInfo();
472
514473 // First allocate registers for the empty intervals.
515474 for (RegSet::const_iterator
516 itr = emptyIntervalVRegs.begin(), end = emptyIntervalVRegs.end();
517 itr != end; ++itr) {
518 LiveInterval *li = &lis->getInterval(*itr);
519
520 unsigned physReg = mri->getSimpleHint(li->reg);
521
522 if (physReg == 0) {
523 const TargetRegisterClass *liRC = mri->getRegClass(li->reg);
524 physReg = liRC->getRawAllocationOrder(*mf).front();
525 }
526
527 vrm->assignVirt2Phys(li->reg, physReg);
475 I = EmptyIntervalVRegs.begin(), E = EmptyIntervalVRegs.end();
476 I != E; ++I) {
477 LiveInterval &LI = LIS.getInterval(*I);
478
479 unsigned PReg = MRI.getSimpleHint(LI.reg);
480
481 if (PReg == 0) {
482 const TargetRegisterClass &RC = *MRI.getRegClass(LI.reg);
483 PReg = RC.getRawAllocationOrder(MF).front();
484 }
485
486 VRM.assignVirt2Phys(LI.reg, PReg);
528487 }
529488 }
530489
531490 bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) {
532
533 mf = &MF;
534 tm = &mf->getTarget();
535 tri = tm->getSubtargetImpl()->getRegisterInfo();
536 tii = tm->getSubtargetImpl()->getInstrInfo();
537 mri = &mf->getRegInfo();
538
539 lis = &getAnalysis();
540 lss = &getAnalysis();
541 mbfi = &getAnalysis();
542
543 calculateSpillWeightsAndHints(*lis, MF, getAnalysis(),
544 *mbfi);
545
546 vrm = &getAnalysis();
547 spiller.reset(createInlineSpiller(*this, MF, *vrm));
548
549 mri->freezeReservedRegs(MF);
550
551 DEBUG(dbgs() << "PBQP Register Allocating for " << mf->getName() << "\n");
491 LiveIntervals &LIS = getAnalysis();
492 MachineBlockFrequencyInfo &MBFI =
493 getAnalysis();
494
495 calculateSpillWeightsAndHints(LIS, MF, getAnalysis(), MBFI);
496
497 VirtRegMap &VRM = getAnalysis();
498
499 std::unique_ptr VRegSpiller(createInlineSpiller(*this, MF, VRM));
500
501 MF.getRegInfo().freezeReservedRegs(MF);
502
503 DEBUG(dbgs() << "PBQP Register Allocating for " << MF.getName() << "\n");
552504
553505 // Allocator main loop:
554506 //
560512 // This process is continued till no more spills are generated.
561513
562514 // Find the vreg intervals in need of allocation.
563 findVRegIntervalsToAlloc();
515 findVRegIntervalsToAlloc(MF, LIS);
564516
565517 #ifndef NDEBUG
566 const Function* func = mf->getFunction();
567 std::string fqn =
568 func->getParent()->getModuleIdentifier() + "." +
569 func->getName().str();
518 const Function &F = *MF.getFunction();
519 std::string FullyQualifiedName =
520 F.getParent()->getModuleIdentifier() + "." + F.getName().str();
570521 #endif
571522
572523 // If there are non-empty intervals allocate them using pbqp.
573 if (!vregsToAlloc.empty()) {
574
575 bool pbqpAllocComplete = false;
576 unsigned round = 0;
577
578 while (!pbqpAllocComplete) {
579 DEBUG(dbgs() << " PBQP Regalloc round " << round << ":\n");
580
581 std::unique_ptr problem =
582 builder->build(mf, lis, mbfi, vregsToAlloc);
524 if (!VRegsToAlloc.empty()) {
525
526 const TargetSubtargetInfo &Subtarget = *MF.getTarget().getSubtargetImpl();
527 std::unique_ptr ConstraintsRoot =
528 llvm::make_unique();
529 ConstraintsRoot->addConstraint(llvm::make_unique());
530 ConstraintsRoot->addConstraint(llvm::make_unique());
531 if (PBQPCoalescing)
532 ConstraintsRoot->addConstraint(llvm::make_unique());
533 ConstraintsRoot->addConstraint(Subtarget.getCustomPBQPConstraints());
534
535 bool PBQPAllocComplete = false;
536 unsigned Round = 0;
537
538 while (!PBQPAllocComplete) {
539 DEBUG(dbgs() << " PBQP Regalloc round " << Round << ":\n");
540
541 PBQPRAGraph G(PBQPRAGraph::GraphMetadata(MF, LIS, MBFI));
542 initializeGraph(G);
543 ConstraintsRoot->apply(G);
583544
584545 #ifndef NDEBUG
585 if (pbqpDumpGraphs) {
586 std::ostringstream rs;
587 rs << round;
588 std::string graphFileName(fqn + "." + rs.str() + ".pbqpgraph");
546 if (PBQPDumpGraphs) {
547 std::ostringstream RS;
548 RS << Round;
549 std::string GraphFileName = FullyQualifiedName + "." + RS.str() +
550 ".pbqpgraph";
589551 std::error_code EC;
590 raw_fd_ostream os(graphFileName, EC, sys::fs::F_Text);
591 DEBUG(dbgs() << "Dumping graph for round " << round << " to \""
592 << graphFileName << "\"\n");
593 problem->getGraph().dump(os);
552 raw_fd_ostream OS(GraphFileName, EC, sys::fs::F_Text);
553 DEBUG(dbgs() << "Dumping graph for round " << Round << " to \""
554 << GraphFileName << "\"\n");
555 G.dumpToStream(OS);
594556 }
595557 #endif
596558
597 PBQP::Solution solution =
598 PBQP::RegAlloc::solve(problem->getGraph());
599
600 pbqpAllocComplete = mapPBQPToRegAlloc(*problem, solution);
601
602 ++round;
559 PBQP::Solution Solution = PBQP::RegAlloc::solve(G);
560 PBQPAllocComplete = mapPBQPToRegAlloc(G, Solution, VRM, *VRegSpiller);
561 ++Round;
603562 }
604563 }
605564
606565 // Finalise allocation, allocate empty ranges.
607 finalizeAlloc();
608 vregsToAlloc.clear();
609 emptyIntervalVRegs.clear();
610
611 DEBUG(dbgs() << "Post alloc VirtRegMap:\n" << *vrm << "\n");
566 finalizeAlloc(MF, LIS, VRM);
567 VRegsToAlloc.clear();
568 EmptyIntervalVRegs.clear();
569
570 DEBUG(dbgs() << "Post alloc VirtRegMap:\n" << VRM << "\n");
612571
613572 return true;
614573 }
615574
616 FunctionPass *
617 llvm::createPBQPRegisterAllocator(std::unique_ptr builder,
618 char *customPassID) {
619 return new RegAllocPBQP(std::move(builder), customPassID);
575 FunctionPass *llvm::createPBQPRegisterAllocator(char *customPassID) {
576 return new RegAllocPBQP(customPassID);
620577 }
621578
622579 FunctionPass* llvm::createDefaultPBQPRegisterAllocator() {
623 std::unique_ptr Builder;
624 if (pbqpCoalescing)
625 Builder = llvm::make_unique();
626 else
627 Builder = llvm::make_unique();
628 return createPBQPRegisterAllocator(std::move(Builder));
580 return createPBQPRegisterAllocator();
629581 }
630582
631583 #undef DEBUG_TYPE
3838 FunctionPass *createAArch64ConditionOptimizerPass();
3939 FunctionPass *createAArch64AddressTypePromotionPass();
4040 FunctionPass *createAArch64A57FPLoadBalancing();
41 FunctionPass *createAArch64A57PBQPRegAlloc();
4241 /// \brief Creates an ARM-specific Target Transformation Info pass.
4342 ImmutablePass *
4443 createAArch64TargetTransformInfoPass(const AArch64TargetMachine *TM);
1717 #define DEBUG_TYPE "aarch64-pbqp"
1818
1919 #include "AArch64.h"
20 #include "AArch64PBQPRegAlloc.h"
2021 #include "AArch64RegisterInfo.h"
21
22 #include "llvm/ADT/SetVector.h"
2322 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
2423 #include "llvm/CodeGen/MachineBasicBlock.h"
2524 #include "llvm/CodeGen/MachineFunction.h"
2827 #include "llvm/Support/Debug.h"
2928 #include "llvm/Support/ErrorHandling.h"
3029 #include "llvm/Support/raw_ostream.h"
31
32 #define PBQP_BUILDER PBQPBuilderWithCoalescing
3330
3431 using namespace llvm;
3532
156153 return isOdd(reg1) == isOdd(reg2);
157154 }
158155
159 class A57PBQPBuilder : public PBQP_BUILDER {
160 public:
161 A57PBQPBuilder() : PBQP_BUILDER(), TRI(nullptr), LIs(nullptr), Chains() {}
162
163 // Build a PBQP instance to represent the register allocation problem for
164 // the given MachineFunction.
165 std::unique_ptr
166 build(MachineFunction *MF, const LiveIntervals *LI,
167 const MachineBlockFrequencyInfo *blockInfo,
168 const RegSet &VRegs) override;
169
170 private:
171 const AArch64RegisterInfo *TRI;
172 const LiveIntervals *LIs;
173 SmallSetVector Chains;
174
175 // Return true if reg is a physical register
176 bool isPhysicalReg(unsigned reg) const {
177 return TRI->isPhysicalRegister(reg);
178 }
179
180 // Add the accumulator chaining constraint, inside the chain, i.e. so that
181 // parity(Rd) == parity(Ra).
182 // \return true if a constraint was added
183 bool addIntraChainConstraint(PBQPRAProblem *p, unsigned Rd, unsigned Ra);
184
185 // Add constraints between existing chains
186 void addInterChainConstraint(PBQPRAProblem *p, unsigned Rd, unsigned Ra);
187 };
188 } // Anonymous namespace
189
190 bool A57PBQPBuilder::addIntraChainConstraint(PBQPRAProblem *p, unsigned Rd,
191 unsigned Ra) {
156 }
157
158 bool A57PBQPConstraints::addIntraChainConstraint(PBQPRAGraph &G, unsigned Rd,
159 unsigned Ra) {
192160 if (Rd == Ra)
193161 return false;
194162
195 if (isPhysicalReg(Rd) || isPhysicalReg(Ra)) {
196 DEBUG(dbgs() << "Rd is a physical reg:" << isPhysicalReg(Rd) << '\n');
197 DEBUG(dbgs() << "Ra is a physical reg:" << isPhysicalReg(Ra) << '\n');
163 const TargetRegisterInfo &TRI =
164 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
165 LiveIntervals &LIs = G.getMetadata().LIS;
166
167 if (TRI.isPhysicalRegister(Rd) || TRI.isPhysicalRegister(Ra)) {
168 DEBUG(dbgs() << "Rd is a physical reg:" << TRI.isPhysicalRegister(Rd)
169 << '\n');
170 DEBUG(dbgs() << "Ra is a physical reg:" << TRI.isPhysicalRegister(Ra)
171 << '\n');
198172 return false;
199173 }
200174
201 const PBQPRAProblem::AllowedSet *vRdAllowed = &p->getAllowedSet(Rd);
202 const PBQPRAProblem::AllowedSet *vRaAllowed = &p->getAllowedSet(Ra);
203
204 PBQPRAGraph &g = p->getGraph();
205 PBQPRAGraph::NodeId node1 = p->getNodeForVReg(Rd);
206 PBQPRAGraph::NodeId node2 = p->getNodeForVReg(Ra);
207 PBQPRAGraph::EdgeId edge = g.findEdge(node1, node2);
175 PBQPRAGraph::NodeId node1 = G.getMetadata().getNodeIdForVReg(Rd);
176 PBQPRAGraph::NodeId node2 = G.getMetadata().getNodeIdForVReg(Ra);
177
178 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRdAllowed =
179 &G.getNodeMetadata(node1).getOptionRegs();
180 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRaAllowed =
181 &G.getNodeMetadata(node2).getOptionRegs();
182
183 PBQPRAGraph::EdgeId edge = G.findEdge(node1, node2);
208184
209185 // The edge does not exist. Create one with the appropriate interference
210186 // costs.
211 if (edge == g.invalidEdgeId()) {
212 const LiveInterval &ld = LIs->getInterval(Rd);
213 const LiveInterval &la = LIs->getInterval(Ra);
187 if (edge == G.invalidEdgeId()) {
188 const LiveInterval &ld = LIs.getInterval(Rd);
189 const LiveInterval &la = LIs.getInterval(Ra);
214190 bool livesOverlap = ld.overlaps(la);
215191
216 PBQP::Matrix costs(vRdAllowed->size() + 1, vRaAllowed->size() + 1, 0);
192 PBQPRAGraph::RawMatrix costs(vRdAllowed->size() + 1,
193 vRaAllowed->size() + 1, 0);
217194 for (unsigned i = 0, ie = vRdAllowed->size(); i != ie; ++i) {
218195 unsigned pRd = (*vRdAllowed)[i];
219196 for (unsigned j = 0, je = vRaAllowed->size(); j != je; ++j) {
220197 unsigned pRa = (*vRaAllowed)[j];
221 if (livesOverlap && TRI->regsOverlap(pRd, pRa))
198 if (livesOverlap && TRI.regsOverlap(pRd, pRa))
222199 costs[i + 1][j + 1] = std::numeric_limits::infinity();
223200 else
224201 costs[i + 1][j + 1] = haveSameParity(pRd, pRa) ? 0.0 : 1.0;
225202 }
226203 }
227 g.addEdge(node1, node2, std::move(costs));
204 G.addEdge(node1, node2, std::move(costs));
228205 return true;
229206 }
230207
231 if (g.getEdgeNode1Id(edge) == node2) {
208 if (G.getEdgeNode1Id(edge) == node2) {
232209 std::swap(node1, node2);
233210 std::swap(vRdAllowed, vRaAllowed);
234211 }
235212
236213 // Enforce minCost(sameParity(RaClass)) > maxCost(otherParity(RdClass))
237 PBQP::Matrix costs(g.getEdgeCosts(edge));
214 PBQPRAGraph::RawMatrix costs(G.getEdgeCosts(edge));
238215 for (unsigned i = 0, ie = vRdAllowed->size(); i != ie; ++i) {
239216 unsigned pRd = (*vRdAllowed)[i];
240217
259236 costs[i + 1][j + 1] = sameParityMax + 1.0;
260237 }
261238 }
262 g.setEdgeCosts(edge, costs);
239 G.setEdgeCosts(edge, std::move(costs));
263240
264241 return true;
265242 }
266243
267 void
268 A57PBQPBuilder::addInterChainConstraint(PBQPRAProblem *p, unsigned Rd,
269 unsigned Ra) {
244 void A57PBQPConstraints::addInterChainConstraint(PBQPRAGraph &G, unsigned Rd,
245 unsigned Ra) {
246 const TargetRegisterInfo &TRI =
247 *G.getMetadata().MF.getTarget().getSubtargetImpl()->getRegisterInfo();
248 (void)TRI;
249 LiveIntervals &LIs = G.getMetadata().LIS;
250
270251 // Do some Chain management
271252 if (Chains.count(Ra)) {
272253 if (Rd != Ra) {
273 DEBUG(dbgs() << "Moving acc chain from " << PrintReg(Ra, TRI) << " to "
274 << PrintReg(Rd, TRI) << '\n';);
254 DEBUG(dbgs() << "Moving acc chain from " << PrintReg(Ra, &TRI) << " to "
255 << PrintReg(Rd, &TRI) << '\n';);
275256 Chains.remove(Ra);
276257 Chains.insert(Rd);
277258 }
278259 } else {
279 DEBUG(dbgs() << "Creating new acc chain for " << PrintReg(Rd, TRI)
260 DEBUG(dbgs() << "Creating new acc chain for " << PrintReg(Rd, &TRI)
280261 << '\n';);
281262 Chains.insert(Rd);
282263 }
283264
284 const LiveInterval &ld = LIs->getInterval(Rd);
265 PBQPRAGraph::NodeId node1 = G.getMetadata().getNodeIdForVReg(Rd);
266
267 const LiveInterval &ld = LIs.getInterval(Rd);
285268 for (auto r : Chains) {
286269 // Skip self
287270 if (r == Rd)
288271 continue;
289272
290 const LiveInterval &lr = LIs->getInterval(r);
273 const LiveInterval &lr = LIs.getInterval(r);
291274 if (ld.overlaps(lr)) {
292 const PBQPRAProblem::AllowedSet *vRdAllowed = &p->getAllowedSet(Rd);
293 const PBQPRAProblem::AllowedSet *vRrAllowed = &p->getAllowedSet(r);
294
295 PBQPRAGraph &g = p->getGraph();
296 PBQPRAGraph::NodeId node1 = p->getNodeForVReg(Rd);
297 PBQPRAGraph::NodeId node2 = p->getNodeForVReg(r);
298 PBQPRAGraph::EdgeId edge = g.findEdge(node1, node2);
299 assert(edge != g.invalidEdgeId() &&
275 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRdAllowed =
276 &G.getNodeMetadata(node1).getOptionRegs();
277
278 PBQPRAGraph::NodeId node2 = G.getMetadata().getNodeIdForVReg(r);
279 const PBQPRAGraph::NodeMetadata::OptionToRegMap *vRrAllowed =
280 &G.getNodeMetadata(node2).getOptionRegs();
281
282 PBQPRAGraph::EdgeId edge = G.findEdge(node1, node2);
283 assert(edge != G.invalidEdgeId() &&
300284 "PBQP error ! The edge should exist !");
301285
302286 DEBUG(dbgs() << "Refining constraint !\n";);
303287
304 if (g.getEdgeNode1Id(edge) == node2) {
288 if (G.getEdgeNode1Id(edge) == node2) {
305289 std::swap(node1, node2);
306290 std::swap(vRdAllowed, vRrAllowed);
307291 }
308292
309293 // Enforce that cost is higher with all other Chains of the same parity
310 PBQP::Matrix costs(g.getEdgeCosts(edge));
294 PBQP::Matrix costs(G.getEdgeCosts(edge));
311295 for (unsigned i = 0, ie = vRdAllowed->size(); i != ie; ++i) {
312296 unsigned pRd = (*vRdAllowed)[i];
313297
332316 costs[i + 1][j + 1] = sameParityMax + 1.0;
333317 }
334318 }
335 g.setEdgeCosts(edge, costs);
336 }
337 }
338 }
339
340 std::unique_ptr
341 A57PBQPBuilder::build(MachineFunction *MF, const LiveIntervals *LI,
342 const MachineBlockFrequencyInfo *blockInfo,
343 const RegSet &VRegs) {
344 std::unique_ptr p =
345 PBQP_BUILDER::build(MF, LI, blockInfo, VRegs);
346
347 TRI = static_cast(
348 MF->getTarget().getSubtargetImpl()->getRegisterInfo());
349 LIs = LI;
350
351 DEBUG(MF->dump(););
352
353 for (MachineFunction::const_iterator mbbItr = MF->begin(), mbbEnd = MF->end();
319 G.setEdgeCosts(edge, std::move(costs));
320 }
321 }
322 }
323
324 void A57PBQPConstraints::apply(PBQPRAGraph &G) {
325 MachineFunction &MF = G.getMetadata().MF;
326
327 const TargetRegisterInfo &TRI =
328 *MF.getTarget().getSubtargetImpl()->getRegisterInfo();
329 (void)TRI;
330 DEBUG(MF.dump());
331
332 for (MachineFunction::const_iterator mbbItr = MF.begin(), mbbEnd = MF.end();
354333 mbbItr != mbbEnd; ++mbbItr) {
355334 const MachineBasicBlock *MBB = &*mbbItr;
356335 Chains.clear(); // FIXME: really needed ? Could not work at MF level ?
371350 unsigned Rd = MI->getOperand(0).getReg();
372351 unsigned Ra = MI->getOperand(3).getReg();
373352
374 if (addIntraChainConstraint(p.get(), Rd, Ra))
375 addInterChainConstraint(p.get(), Rd, Ra);
353 if (addIntraChainConstraint(G, Rd, Ra))
354 addInterChainConstraint(G, Rd, Ra);
376355 break;
377356 }
378357
379358 case AArch64::FMLAv2f32:
380359 case AArch64::FMLSv2f32: {
381360 unsigned Rd = MI->getOperand(0).getReg();
382 addInterChainConstraint(p.get(), Rd, Rd);
361 addInterChainConstraint(G, Rd, Rd);
383362 break;
384363 }
385364
388367 for (auto r : Chains) {
389368 SmallVector toDel;
390369 if (MI->killsRegister(r)) {
391 DEBUG(dbgs() << "Killing chain " << PrintReg(r, TRI) << " at ";
370 DEBUG(dbgs() << "Killing chain " << PrintReg(r, &TRI) << " at ";
392371 MI->print(dbgs()););
393372 toDel.push_back(r);
394373 }
401380 }
402381 }
403382 }
404
405 return p;
406 }
407
408 // Factory function used by AArch64TargetMachine to add the pass to the
409 // passmanager.
410 FunctionPass *llvm::createAArch64A57PBQPRegAlloc() {
411 std::unique_ptr builder = llvm::make_unique();
412 return createPBQPRegisterAllocator(std::move(builder), nullptr);
413 }
383 }
1111 //===----------------------------------------------------------------------===//
1212
1313 #include "AArch64InstrInfo.h"
14 #include "AArch64PBQPRegAlloc.h"
1415 #include "AArch64Subtarget.h"
1516 #include "llvm/ADT/SmallVector.h"
1617 #include "llvm/CodeGen/MachineScheduler.h"
132133 bool AArch64Subtarget::enableEarlyIfConversion() const {
133134 return EnableEarlyIfConvert;
134135 }
136
137 std::unique_ptr
138 AArch64Subtarget::getCustomPBQPConstraints() const {
139 return llvm::make_unique();
140 }
141141 unsigned NumRegionInstrs) const override;
142142
143143 bool enableEarlyIfConversion() const override;
144
145 std::unique_ptr getCustomPBQPConstraints() const override;
144146 };
145147 } // End llvm namespace
146148
101101
102102 if (EnablePBQP && Subtarget.isCortexA57() && OL != CodeGenOpt::None) {
103103 usingPBQP = true;
104 RegisterRegAlloc::setDefault(createAArch64A57PBQPRegAlloc);
104 RegisterRegAlloc::setDefault(createDefaultPBQPRegisterAllocator);
105105 }
106106 }
107107