llvm.org GIT mirror llvm / c205b62
Move AArch64BranchRelaxation to generic code git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@283459 91177308-0d34-0410-b5e6-96231b3b80d8 Matt Arsenault 3 years ago
12 changed file(s) with 418 addition(s) and 424 deletion(s). Raw diff Collapse all Expand all
179179 /// successor blocks (creating fall throughs), and eliminating branches over
180180 /// branches.
181181 extern char &BranchFolderPassID;
182
183 /// BranchRelaxation - This pass replaces branches that need to jump further
184 /// than is supported by a branch instruction.
185 extern char &BranchRelaxationPassID;
182186
183187 /// MachineFunctionPrinterPass - This pass prints out MachineInstr's.
184188 extern char &MachineFunctionPrinterPassID;
7878 void initializeBoundsCheckingPass(PassRegistry&);
7979 void initializeBranchFolderPassPass(PassRegistry&);
8080 void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
81 void initializeBranchRelaxationPass(PassRegistry&);
8182 void initializeBreakCriticalEdgesPass(PassRegistry&);
8283 void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&);
8384 void initializeCFGPrinterLegacyPassPass(PassRegistry&);
0 //===-- BranchRelaxation.cpp ----------------------------------------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/CodeGen/Passes.h"
10 #include "llvm/ADT/SmallVector.h"
11 #include "llvm/ADT/Statistic.h"
12 #include "llvm/CodeGen/MachineFunctionPass.h"
13 #include "llvm/Target/TargetInstrInfo.h"
14 #include "llvm/Target/TargetSubtargetInfo.h"
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Format.h"
17 #include "llvm/Support/raw_ostream.h"
18
19 using namespace llvm;
20
21 #define DEBUG_TYPE "branch-relaxation"
22
23 STATISTIC(NumSplit, "Number of basic blocks split");
24 STATISTIC(NumConditionalRelaxed, "Number of conditional branches relaxed");
25
26 #define BRANCH_RELAX_NAME "Branch relaxation pass"
27
28 namespace {
29 class BranchRelaxation : public MachineFunctionPass {
30 /// BasicBlockInfo - Information about the offset and size of a single
31 /// basic block.
32 struct BasicBlockInfo {
33 /// Offset - Distance from the beginning of the function to the beginning
34 /// of this basic block.
35 ///
36 /// The offset is always aligned as required by the basic block.
37 unsigned Offset;
38
39 /// Size - Size of the basic block in bytes. If the block contains
40 /// inline assembly, this is a worst case estimate.
41 ///
42 /// The size does not include any alignment padding whether from the
43 /// beginning of the block, or from an aligned jump table at the end.
44 unsigned Size;
45
46 BasicBlockInfo() : Offset(0), Size(0) {}
47
48 /// Compute the offset immediately following this block. If LogAlign is
49 /// specified, return the offset the successor block will get if it has
50 /// this alignment.
51 unsigned postOffset(unsigned LogAlign = 0) const {
52 unsigned PO = Offset + Size;
53 unsigned Align = 1 << LogAlign;
54 return (PO + Align - 1) / Align * Align;
55 }
56 };
57
58 SmallVector BlockInfo;
59
60 MachineFunction *MF;
61 const TargetInstrInfo *TII;
62
63 bool relaxBranchInstructions();
64 void scanFunction();
65 MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI);
66 void adjustBlockOffsets(MachineBasicBlock &MBB);
67 bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const;
68
69 bool fixupConditionalBranch(MachineInstr &MI);
70 uint64_t computeBlockSize(const MachineBasicBlock &MBB) const;
71 unsigned getInstrOffset(const MachineInstr &MI) const;
72 void dumpBBs();
73 void verify();
74
75 public:
76 static char ID;
77 BranchRelaxation() : MachineFunctionPass(ID) { }
78
79 bool runOnMachineFunction(MachineFunction &MF) override;
80
81 StringRef getPassName() const override {
82 return BRANCH_RELAX_NAME;
83 }
84 };
85
86 }
87
88 char BranchRelaxation::ID = 0;
89 char &llvm::BranchRelaxationPassID = BranchRelaxation::ID;
90
91 INITIALIZE_PASS(BranchRelaxation, DEBUG_TYPE, BRANCH_RELAX_NAME, false, false)
92
93 /// verify - check BBOffsets, BBSizes, alignment of islands
94 void BranchRelaxation::verify() {
95 #ifndef NDEBUG
96 unsigned PrevNum = MF->begin()->getNumber();
97 for (MachineBasicBlock &MBB : *MF) {
98 unsigned Align = MBB.getAlignment();
99 unsigned Num = MBB.getNumber();
100 assert(BlockInfo[Num].Offset % (1u << Align) == 0);
101 assert(!Num || BlockInfo[PrevNum].postOffset() <= BlockInfo[Num].Offset);
102 PrevNum = Num;
103 }
104 #endif
105 }
106
107 /// print block size and offset information - debugging
108 void BranchRelaxation::dumpBBs() {
109 for (auto &MBB : *MF) {
110 const BasicBlockInfo &BBI = BlockInfo[MBB.getNumber()];
111 dbgs() << format("BB#%u\toffset=%08x\t", MBB.getNumber(), BBI.Offset)
112 << format("size=%#x\n", BBI.Size);
113 }
114 }
115
116 /// scanFunction - Do the initial scan of the function, building up
117 /// information about each block.
118 void BranchRelaxation::scanFunction() {
119 BlockInfo.clear();
120 BlockInfo.resize(MF->getNumBlockIDs());
121
122 // First thing, compute the size of all basic blocks, and see if the function
123 // has any inline assembly in it. If so, we have to be conservative about
124 // alignment assumptions, as we don't know for sure the size of any
125 // instructions in the inline assembly.
126 for (MachineBasicBlock &MBB : *MF)
127 BlockInfo[MBB.getNumber()].Size = computeBlockSize(MBB);
128
129 // Compute block offsets and known bits.
130 adjustBlockOffsets(*MF->begin());
131 }
132
133 /// computeBlockSize - Compute the size for MBB.
134 uint64_t BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) const {
135 uint64_t Size = 0;
136 for (const MachineInstr &MI : MBB)
137 Size += TII->getInstSizeInBytes(MI);
138 return Size;
139 }
140
141 /// getInstrOffset - Return the current offset of the specified machine
142 /// instruction from the start of the function. This offset changes as stuff is
143 /// moved around inside the function.
144 unsigned BranchRelaxation::getInstrOffset(const MachineInstr &MI) const {
145 const MachineBasicBlock *MBB = MI.getParent();
146
147 // The offset is composed of two things: the sum of the sizes of all MBB's
148 // before this instruction's block, and the offset from the start of the block
149 // it is in.
150 unsigned Offset = BlockInfo[MBB->getNumber()].Offset;
151
152 // Sum instructions before MI in MBB.
153 for (MachineBasicBlock::const_iterator I = MBB->begin(); &*I != &MI; ++I) {
154 assert(I != MBB->end() && "Didn't find MI in its own basic block?");
155 Offset += TII->getInstSizeInBytes(*I);
156 }
157
158 return Offset;
159 }
160
161 void BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start) {
162 unsigned PrevNum = Start.getNumber();
163 for (auto &MBB : make_range(MachineFunction::iterator(Start), MF->end())) {
164 unsigned Num = MBB.getNumber();
165 if (!Num) // block zero is never changed from offset zero.
166 continue;
167 // Get the offset and known bits at the end of the layout predecessor.
168 // Include the alignment of the current block.
169 unsigned LogAlign = MBB.getAlignment();
170 BlockInfo[Num].Offset = BlockInfo[PrevNum].postOffset(LogAlign);
171 PrevNum = Num;
172 }
173 }
174
175 /// Split the basic block containing MI into two blocks, which are joined by
176 /// an unconditional branch. Update data structures and renumber blocks to
177 /// account for this change and returns the newly created block.
178 /// NOTE: Successor list of the original BB is out of date after this function,
179 /// and must be updated by the caller! Other transforms follow using this
180 /// utility function, so no point updating now rather than waiting.
181 MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI) {
182 MachineBasicBlock *OrigBB = MI.getParent();
183
184 // Create a new MBB for the code after the OrigBB.
185 MachineBasicBlock *NewBB =
186 MF->CreateMachineBasicBlock(OrigBB->getBasicBlock());
187 MF->insert(++OrigBB->getIterator(), NewBB);
188
189 // Splice the instructions starting with MI over to NewBB.
190 NewBB->splice(NewBB->end(), OrigBB, MI.getIterator(), OrigBB->end());
191
192 // Add an unconditional branch from OrigBB to NewBB.
193 // Note the new unconditional branch is not being recorded.
194 // There doesn't seem to be meaningful DebugInfo available; this doesn't
195 // correspond to anything in the source.
196 TII->insertUnconditionalBranch(*OrigBB, NewBB, DebugLoc());
197
198 // Insert an entry into BlockInfo to align it properly with the block numbers.
199 BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
200
201 // Figure out how large the OrigBB is. As the first half of the original
202 // block, it cannot contain a tablejump. The size includes
203 // the new jump we added. (It should be possible to do this without
204 // recounting everything, but it's very confusing, and this is rarely
205 // executed.)
206 BlockInfo[OrigBB->getNumber()].Size = computeBlockSize(*OrigBB);
207
208 // Figure out how large the NewMBB is. As the second half of the original
209 // block, it may contain a tablejump.
210 BlockInfo[NewBB->getNumber()].Size = computeBlockSize(*NewBB);
211
212 // All BBOffsets following these blocks must be modified.
213 adjustBlockOffsets(*OrigBB);
214
215 ++NumSplit;
216
217 return NewBB;
218 }
219
220 /// isBlockInRange - Returns true if the distance between specific MI and
221 /// specific BB can fit in MI's displacement field.
222 bool BranchRelaxation::isBlockInRange(
223 const MachineInstr &MI, const MachineBasicBlock &DestBB) const {
224 int64_t BrOffset = getInstrOffset(MI);
225 int64_t DestOffset = BlockInfo[DestBB.getNumber()].Offset;
226
227 if (TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - BrOffset))
228 return true;
229
230 DEBUG(
231 dbgs() << "Out of range branch to destination BB#" << DestBB.getNumber()
232 << " from BB#" << MI.getParent()->getNumber()
233 << " to " << DestOffset
234 << " offset " << DestOffset - BrOffset
235 << '\t' << MI
236 );
237
238 return false;
239 }
240
241 /// fixupConditionalBranch - Fix up a conditional branch whose destination is
242 /// too far away to fit in its displacement field. It is converted to an inverse
243 /// conditional branch + an unconditional branch to the destination.
244 bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
245 DebugLoc DL = MI.getDebugLoc();
246 MachineBasicBlock *MBB = MI.getParent();
247 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
248 SmallVector Cond;
249
250 bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
251 assert(!Fail && "branches to be relaxed must be analyzable");
252 (void)Fail;
253
254 // Add an unconditional branch to the destination and invert the branch
255 // condition to jump over it:
256 // tbz L1
257 // =>
258 // tbnz L2
259 // b L1
260 // L2:
261
262 if (FBB && isBlockInRange(MI, *FBB)) {
263 // Last MI in the BB is an unconditional branch. We can simply invert the
264 // condition and swap destinations:
265 // beq L1
266 // b L2
267 // =>
268 // bne L2
269 // b L1
270 DEBUG(dbgs() << " Invert condition and swap "
271 "its destination with " << MBB->back());
272
273 TII->reverseBranchCondition(Cond);
274 int OldSize = 0, NewSize = 0;
275 TII->removeBranch(*MBB, &OldSize);
276 TII->insertBranch(*MBB, FBB, TBB, Cond, DL, &NewSize);
277
278 BlockInfo[MBB->getNumber()].Size += (NewSize - OldSize);
279 return true;
280 } else if (FBB) {
281 // We need to split the basic block here to obtain two long-range
282 // unconditional branches.
283 auto &NewBB = *MF->CreateMachineBasicBlock(MBB->getBasicBlock());
284 MF->insert(++MBB->getIterator(), &NewBB);
285
286 // Insert an entry into BlockInfo to align it properly with the block
287 // numbers.
288 BlockInfo.insert(BlockInfo.begin() + NewBB.getNumber(), BasicBlockInfo());
289
290 unsigned &NewBBSize = BlockInfo[NewBB.getNumber()].Size;
291 int NewBrSize;
292 TII->insertUnconditionalBranch(NewBB, FBB, DL, &NewBrSize);
293 NewBBSize += NewBrSize;
294
295 // Update the successor lists according to the transformation to follow.
296 // Do it here since if there's no split, no update is needed.
297 MBB->replaceSuccessor(FBB, &NewBB);
298 NewBB.addSuccessor(FBB);
299 }
300
301 // We now have an appropriate fall-through block in place (either naturally or
302 // just created), so we can invert the condition.
303 MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
304
305 DEBUG(dbgs() << " Insert B to BB#" << TBB->getNumber()
306 << ", invert condition and change dest. to BB#"
307 << NextBB.getNumber() << '\n');
308
309 unsigned &MBBSize = BlockInfo[MBB->getNumber()].Size;
310
311 // Insert a new conditional branch and a new unconditional branch.
312 int RemovedSize = 0;
313 TII->reverseBranchCondition(Cond);
314 TII->removeBranch(*MBB, &RemovedSize);
315 MBBSize -= RemovedSize;
316
317 int AddedSize = 0;
318 TII->insertBranch(*MBB, &NextBB, TBB, Cond, DL, &AddedSize);
319 MBBSize += AddedSize;
320
321 // Finally, keep the block offsets up to date.
322 adjustBlockOffsets(*MBB);
323 return true;
324 }
325
326 bool BranchRelaxation::relaxBranchInstructions() {
327 bool Changed = false;
328 // Relaxing branches involves creating new basic blocks, so re-eval
329 // end() for termination.
330 for (MachineFunction::iterator I = MF->begin(); I != MF->end(); ++I) {
331 MachineBasicBlock &MBB = *I;
332 MachineBasicBlock::iterator J = MBB.getFirstTerminator();
333 if (J == MBB.end())
334 continue;
335
336
337 MachineBasicBlock::iterator Next;
338 for (MachineBasicBlock::iterator J = MBB.getFirstTerminator();
339 J != MBB.end(); J = Next) {
340 Next = std::next(J);
341 MachineInstr &MI = *J;
342
343 if (MI.isConditionalBranch()) {
344 MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
345 if (!isBlockInRange(MI, *DestBB)) {
346 if (Next != MBB.end() && Next->isConditionalBranch()) {
347 // If there are multiple conditional branches, this isn't an
348 // analyzable block. Split later terminators into a new block so
349 // each one will be analyzable.
350
351 MachineBasicBlock *NewBB = splitBlockBeforeInstr(*Next);
352 NewBB->transferSuccessors(&MBB);
353 MBB.addSuccessor(NewBB);
354 MBB.addSuccessor(DestBB);
355
356 // Cleanup potential unconditional branch to successor block.
357 NewBB->updateTerminator();
358 MBB.updateTerminator();
359 } else {
360 fixupConditionalBranch(MI);
361 ++NumConditionalRelaxed;
362 }
363
364 Changed = true;
365
366 // This may have modified all of the terminators, so start over.
367 Next = MBB.getFirstTerminator();
368 }
369 }
370 }
371 }
372
373 return Changed;
374 }
375
376 bool BranchRelaxation::runOnMachineFunction(MachineFunction &mf) {
377 MF = &mf;
378
379 DEBUG(dbgs() << "***** BranchRelaxation *****\n");
380
381 TII = MF->getSubtarget().getInstrInfo();
382
383 // Renumber all of the machine basic blocks in the function, guaranteeing that
384 // the numbers agree with the position of the block in the function.
385 MF->RenumberBlocks();
386
387 // Do the initial scan of the function, building up information about the
388 // sizes of each block.
389 scanFunction();
390
391 DEBUG(dbgs() << " Basic blocks before relaxation\n"; dumpBBs(););
392
393 bool MadeChange = false;
394 while (relaxBranchInstructions())
395 MadeChange = true;
396
397 // After a while, this might be made debug-only, but it is not expensive.
398 verify();
399
400 DEBUG(dbgs() << " Basic blocks after relaxation\n\n"; dumpBBs());
401
402 BlockInfo.clear();
403
404 return MadeChange;
405 }
44 AtomicExpandPass.cpp
55 BasicTargetTransformInfo.cpp
66 BranchFolding.cpp
7 BranchRelaxation.cpp
78 BuiltinGCs.cpp
89 CalcSpillWeights.cpp
910 CallingConvLower.cpp
2121 void llvm::initializeCodeGen(PassRegistry &Registry) {
2222 initializeAtomicExpandPass(Registry);
2323 initializeBranchFolderPassPass(Registry);
24 initializeBranchRelaxationPass(Registry);
2425 initializeCodeGenPreparePass(Registry);
2526 initializeCountingFunctionInserterPass(Registry);
2627 initializeDeadMachineInstructionElimPass(Registry);
2929 FunctionPass *createAArch64RedundantCopyEliminationPass();
3030 FunctionPass *createAArch64ConditionalCompares();
3131 FunctionPass *createAArch64AdvSIMDScalar();
32 FunctionPass *createAArch64BranchRelaxation();
3332 FunctionPass *createAArch64ISelDag(AArch64TargetMachine &TM,
3433 CodeGenOpt::Level OptLevel);
3534 FunctionPass *createAArch64StorePairSuppressPass();
4948 void initializeAArch64A57FPLoadBalancingPass(PassRegistry&);
5049 void initializeAArch64AddressTypePromotionPass(PassRegistry&);
5150 void initializeAArch64AdvSIMDScalarPass(PassRegistry&);
52 void initializeAArch64BranchRelaxationPass(PassRegistry&);
5351 void initializeAArch64CollectLOHPass(PassRegistry&);
5452 void initializeAArch64ConditionalComparesPass(PassRegistry&);
5553 void initializeAArch64ConditionOptimizerPass(PassRegistry&);
+0
-416
lib/Target/AArch64/AArch64BranchRelaxation.cpp less more
None //===-- AArch64BranchRelaxation.cpp - AArch64 branch relaxation -----------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "AArch64.h"
10 #include "AArch64InstrInfo.h"
11 #include "AArch64MachineFunctionInfo.h"
12 #include "AArch64Subtarget.h"
13 #include "llvm/ADT/SmallVector.h"
14 #include "llvm/ADT/Statistic.h"
15 #include "llvm/CodeGen/MachineFunctionPass.h"
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/ErrorHandling.h"
18 #include "llvm/Support/Format.h"
19 #include "llvm/Support/raw_ostream.h"
20 using namespace llvm;
21
22 #define DEBUG_TYPE "aarch64-branch-relax"
23
24 STATISTIC(NumSplit, "Number of basic blocks split");
25 STATISTIC(NumConditionalRelaxed, "Number of conditional branches relaxed");
26
27 namespace llvm {
28 void initializeAArch64BranchRelaxationPass(PassRegistry &);
29 }
30
31 #define AARCH64_BR_RELAX_NAME "AArch64 branch relaxation pass"
32
33 namespace {
34 class AArch64BranchRelaxation : public MachineFunctionPass {
35 /// BasicBlockInfo - Information about the offset and size of a single
36 /// basic block.
37 struct BasicBlockInfo {
38 /// Offset - Distance from the beginning of the function to the beginning
39 /// of this basic block.
40 ///
41 /// The offset is always aligned as required by the basic block.
42 unsigned Offset;
43
44 /// Size - Size of the basic block in bytes. If the block contains
45 /// inline assembly, this is a worst case estimate.
46 ///
47 /// The size does not include any alignment padding whether from the
48 /// beginning of the block, or from an aligned jump table at the end.
49 unsigned Size;
50
51 BasicBlockInfo() : Offset(0), Size(0) {}
52
53 /// Compute the offset immediately following this block. If LogAlign is
54 /// specified, return the offset the successor block will get if it has
55 /// this alignment.
56 unsigned postOffset(unsigned LogAlign = 0) const {
57 unsigned PO = Offset + Size;
58 unsigned Align = 1 << LogAlign;
59 return (PO + Align - 1) / Align * Align;
60 }
61 };
62
63 SmallVector BlockInfo;
64
65 MachineFunction *MF;
66 const AArch64InstrInfo *TII;
67
68 bool relaxBranchInstructions();
69 void scanFunction();
70 MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI);
71 void adjustBlockOffsets(MachineBasicBlock &MBB);
72 bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const;
73
74 bool fixupConditionalBranch(MachineInstr &MI);
75 void computeBlockSize(const MachineBasicBlock &MBB);
76 unsigned getInstrOffset(const MachineInstr &MI) const;
77 void dumpBBs();
78 void verify();
79
80 public:
81 static char ID;
82 AArch64BranchRelaxation() : MachineFunctionPass(ID) {
83 initializeAArch64BranchRelaxationPass(*PassRegistry::getPassRegistry());
84 }
85
86 bool runOnMachineFunction(MachineFunction &MF) override;
87
88 StringRef getPassName() const override { return AARCH64_BR_RELAX_NAME; }
89 };
90 char AArch64BranchRelaxation::ID = 0;
91 }
92
93 INITIALIZE_PASS(AArch64BranchRelaxation, "aarch64-branch-relax",
94 AARCH64_BR_RELAX_NAME, false, false)
95
96 /// verify - check BBOffsets, BBSizes, alignment of islands
97 void AArch64BranchRelaxation::verify() {
98 #ifndef NDEBUG
99 unsigned PrevNum = MF->begin()->getNumber();
100 for (MachineBasicBlock &MBB : *MF) {
101 unsigned Align = MBB.getAlignment();
102 unsigned Num = MBB.getNumber();
103 assert(BlockInfo[Num].Offset % (1u << Align) == 0);
104 assert(!Num || BlockInfo[PrevNum].postOffset() <= BlockInfo[Num].Offset);
105 PrevNum = Num;
106 }
107 #endif
108 }
109
110 /// print block size and offset information - debugging
111 void AArch64BranchRelaxation::dumpBBs() {
112 for (auto &MBB : *MF) {
113 const BasicBlockInfo &BBI = BlockInfo[MBB.getNumber()];
114 dbgs() << format("BB#%u\toffset=%08x\t", MBB.getNumber(), BBI.Offset)
115 << format("size=%#x\n", BBI.Size);
116 }
117 }
118
119 /// scanFunction - Do the initial scan of the function, building up
120 /// information about each block.
121 void AArch64BranchRelaxation::scanFunction() {
122 BlockInfo.clear();
123 BlockInfo.resize(MF->getNumBlockIDs());
124
125 // First thing, compute the size of all basic blocks, and see if the function
126 // has any inline assembly in it. If so, we have to be conservative about
127 // alignment assumptions, as we don't know for sure the size of any
128 // instructions in the inline assembly.
129 for (MachineBasicBlock &MBB : *MF)
130 computeBlockSize(MBB);
131
132 // Compute block offsets and known bits.
133 adjustBlockOffsets(*MF->begin());
134 }
135
136 /// computeBlockSize - Compute the size for MBB.
137 /// This function updates BlockInfo directly.
138 void AArch64BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) {
139 unsigned Size = 0;
140 for (const MachineInstr &MI : MBB)
141 Size += TII->getInstSizeInBytes(MI);
142 BlockInfo[MBB.getNumber()].Size = Size;
143 }
144
145 /// getInstrOffset - Return the current offset of the specified machine
146 /// instruction from the start of the function. This offset changes as stuff is
147 /// moved around inside the function.
148 unsigned AArch64BranchRelaxation::getInstrOffset(const MachineInstr &MI) const {
149 const MachineBasicBlock *MBB = MI.getParent();
150
151 // The offset is composed of two things: the sum of the sizes of all MBB's
152 // before this instruction's block, and the offset from the start of the block
153 // it is in.
154 unsigned Offset = BlockInfo[MBB->getNumber()].Offset;
155
156 // Sum instructions before MI in MBB.
157 for (MachineBasicBlock::const_iterator I = MBB->begin(); &*I != &MI; ++I) {
158 assert(I != MBB->end() && "Didn't find MI in its own basic block?");
159 Offset += TII->getInstSizeInBytes(*I);
160 }
161
162 return Offset;
163 }
164
165 void AArch64BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start) {
166 unsigned PrevNum = Start.getNumber();
167 for (auto &MBB : make_range(MachineFunction::iterator(Start), MF->end())) {
168 unsigned Num = MBB.getNumber();
169 if (!Num) // block zero is never changed from offset zero.
170 continue;
171 // Get the offset and known bits at the end of the layout predecessor.
172 // Include the alignment of the current block.
173 unsigned LogAlign = MBB.getAlignment();
174 BlockInfo[Num].Offset = BlockInfo[PrevNum].postOffset(LogAlign);
175 PrevNum = Num;
176 }
177 }
178
179 /// Split the basic block containing MI into two blocks, which are joined by
180 /// an unconditional branch. Update data structures and renumber blocks to
181 /// account for this change and returns the newly created block.
182 /// NOTE: Successor list of the original BB is out of date after this function,
183 /// and must be updated by the caller! Other transforms follow using this
184 /// utility function, so no point updating now rather than waiting.
185 MachineBasicBlock *
186 AArch64BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI) {
187 MachineBasicBlock *OrigBB = MI.getParent();
188
189 // Create a new MBB for the code after the OrigBB.
190 MachineBasicBlock *NewBB =
191 MF->CreateMachineBasicBlock(OrigBB->getBasicBlock());
192 MF->insert(++OrigBB->getIterator(), NewBB);
193
194 // Splice the instructions starting with MI over to NewBB.
195 NewBB->splice(NewBB->end(), OrigBB, MI.getIterator(), OrigBB->end());
196
197 // Add an unconditional branch from OrigBB to NewBB.
198 // Note the new unconditional branch is not being recorded.
199 // There doesn't seem to be meaningful DebugInfo available; this doesn't
200 // correspond to anything in the source.
201 TII->insertUnconditionalBranch(*OrigBB, NewBB, DebugLoc());
202
203 // Insert an entry into BlockInfo to align it properly with the block numbers.
204 BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
205
206 // Figure out how large the OrigBB is. As the first half of the original
207 // block, it cannot contain a tablejump. The size includes
208 // the new jump we added. (It should be possible to do this without
209 // recounting everything, but it's very confusing, and this is rarely
210 // executed.)
211 computeBlockSize(*OrigBB);
212
213 // Figure out how large the NewMBB is. As the second half of the original
214 // block, it may contain a tablejump.
215 computeBlockSize(*NewBB);
216
217 // All BBOffsets following these blocks must be modified.
218 adjustBlockOffsets(*OrigBB);
219
220 ++NumSplit;
221
222 return NewBB;
223 }
224
225 /// isBlockInRange - Returns true if the distance between specific MI and
226 /// specific BB can fit in MI's displacement field.
227 bool AArch64BranchRelaxation::isBlockInRange(
228 const MachineInstr &MI, const MachineBasicBlock &DestBB) const {
229 int64_t BrOffset = getInstrOffset(MI);
230 int64_t DestOffset = BlockInfo[DestBB.getNumber()].Offset;
231
232 if (TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - BrOffset))
233 return true;
234
235 DEBUG(
236 dbgs() << "Out of range branch to destination BB#" << DestBB.getNumber()
237 << " from BB#" << MI.getParent()->getNumber()
238 << " to " << DestOffset
239 << " offset " << DestOffset - BrOffset
240 << '\t' << MI
241 );
242
243 return false;
244 }
245
246 /// fixupConditionalBranch - Fix up a conditional branch whose destination is
247 /// too far away to fit in its displacement field. It is converted to an inverse
248 /// conditional branch + an unconditional branch to the destination.
249 bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
250 DebugLoc DL = MI.getDebugLoc();
251 MachineBasicBlock *MBB = MI.getParent();
252 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
253 SmallVector Cond;
254
255 bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
256 assert(!Fail && "branches to be relaxed must be analyzable");
257 (void)Fail;
258
259 // Add an unconditional branch to the destination and invert the branch
260 // condition to jump over it:
261 // tbz L1
262 // =>
263 // tbnz L2
264 // b L1
265 // L2:
266
267 if (FBB && isBlockInRange(MI, *FBB)) {
268 // Last MI in the BB is an unconditional branch. We can simply invert the
269 // condition and swap destinations:
270 // beq L1
271 // b L2
272 // =>
273 // bne L2
274 // b L1
275 DEBUG(dbgs() << " Invert condition and swap "
276 "its destination with " << MBB->back());
277
278 TII->reverseBranchCondition(Cond);
279 int OldSize = 0, NewSize = 0;
280 TII->removeBranch(*MBB, &OldSize);
281 TII->insertBranch(*MBB, FBB, TBB, Cond, DL, &NewSize);
282
283 BlockInfo[MBB->getNumber()].Size += (NewSize - OldSize);
284 return true;
285 } else if (FBB) {
286 // We need to split the basic block here to obtain two long-range
287 // unconditional branches.
288 auto &NewBB = *MF->CreateMachineBasicBlock(MBB->getBasicBlock());
289 MF->insert(++MBB->getIterator(), &NewBB);
290
291 // Insert an entry into BlockInfo to align it properly with the block
292 // numbers.
293 BlockInfo.insert(BlockInfo.begin() + NewBB.getNumber(), BasicBlockInfo());
294
295 unsigned &NewBBSize = BlockInfo[NewBB.getNumber()].Size;
296 int NewBrSize;
297 TII->insertUnconditionalBranch(NewBB, FBB, DL, &NewBrSize);
298 NewBBSize += NewBrSize;
299
300 // Update the successor lists according to the transformation to follow.
301 // Do it here since if there's no split, no update is needed.
302 MBB->replaceSuccessor(FBB, &NewBB);
303 NewBB.addSuccessor(FBB);
304 }
305
306 // We now have an appropriate fall-through block in place (either naturally or
307 // just created), so we can invert the condition.
308 MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
309
310 DEBUG(dbgs() << " Insert B to BB#" << TBB->getNumber()
311 << ", invert condition and change dest. to BB#"
312 << NextBB.getNumber() << '\n');
313
314 unsigned &MBBSize = BlockInfo[MBB->getNumber()].Size;
315
316 // Insert a new conditional branch and a new unconditional branch.
317 int RemovedSize = 0;
318 TII->reverseBranchCondition(Cond);
319 TII->removeBranch(*MBB, &RemovedSize);
320 MBBSize -= RemovedSize;
321
322 int AddedSize = 0;
323 TII->insertBranch(*MBB, &NextBB, TBB, Cond, DL, &AddedSize);
324 MBBSize += AddedSize;
325
326 // Finally, keep the block offsets up to date.
327 adjustBlockOffsets(*MBB);
328 return true;
329 }
330
331 bool AArch64BranchRelaxation::relaxBranchInstructions() {
332 bool Changed = false;
333 // Relaxing branches involves creating new basic blocks, so re-eval
334 // end() for termination.
335 for (MachineFunction::iterator I = MF->begin(); I != MF->end(); ++I) {
336 MachineBasicBlock &MBB = *I;
337 MachineBasicBlock::iterator J = MBB.getFirstTerminator();
338 if (J == MBB.end())
339 continue;
340
341 MachineBasicBlock::iterator Next;
342 for (MachineBasicBlock::iterator J = MBB.getFirstTerminator();
343 J != MBB.end(); J = Next) {
344 Next = std::next(J);
345 MachineInstr &MI = *J;
346
347 if (MI.isConditionalBranch()) {
348 MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
349 if (!isBlockInRange(MI, *DestBB)) {
350 if (Next != MBB.end() && Next->isConditionalBranch()) {
351 // If there are multiple conditional branches, this isn't an
352 // analyzable block. Split later terminators into a new block so
353 // each one will be analyzable.
354
355 MachineBasicBlock *NewBB = splitBlockBeforeInstr(*Next);
356 NewBB->transferSuccessors(&MBB);
357 MBB.addSuccessor(NewBB);
358 MBB.addSuccessor(DestBB);
359
360 // Cleanup potential unconditional branch to successor block.
361 NewBB->updateTerminator();
362 MBB.updateTerminator();
363 } else {
364 fixupConditionalBranch(MI);
365 ++NumConditionalRelaxed;
366 }
367
368 Changed = true;
369
370 // This may have modified all of the terminators, so start over.
371 Next = MBB.getFirstTerminator();
372 }
373
374 }
375 }
376 }
377
378 return Changed;
379 }
380
381 bool AArch64BranchRelaxation::runOnMachineFunction(MachineFunction &mf) {
382 MF = &mf;
383
384 DEBUG(dbgs() << "***** AArch64BranchRelaxation *****\n");
385
386 TII = MF->getSubtarget().getInstrInfo();
387
388 // Renumber all of the machine basic blocks in the function, guaranteeing that
389 // the numbers agree with the position of the block in the function.
390 MF->RenumberBlocks();
391
392 // Do the initial scan of the function, building up information about the
393 // sizes of each block.
394 scanFunction();
395
396 DEBUG(dbgs() << " Basic blocks before relaxation\n"; dumpBBs(););
397
398 bool MadeChange = false;
399 while (relaxBranchInstructions())
400 MadeChange = true;
401
402 // After a while, this might be made debug-only, but it is not expensive.
403 verify();
404
405 DEBUG(dbgs() << " Basic blocks after relaxation\n\n"; dumpBBs());
406
407 BlockInfo.clear();
408
409 return MadeChange;
410 }
411
412 /// Returns an instance of the AArch64 Branch Relaxation pass.
413 FunctionPass *llvm::createAArch64BranchRelaxation() {
414 return new AArch64BranchRelaxation();
415 }
134134 initializeAArch64A57FPLoadBalancingPass(*PR);
135135 initializeAArch64AddressTypePromotionPass(*PR);
136136 initializeAArch64AdvSIMDScalarPass(*PR);
137 initializeAArch64BranchRelaxationPass(*PR);
138137 initializeAArch64CollectLOHPass(*PR);
139138 initializeAArch64ConditionalComparesPass(*PR);
140139 initializeAArch64ConditionOptimizerPass(*PR);
462461 // Relax conditional branch instructions if they're otherwise out of
463462 // range of their destination.
464463 if (BranchRelaxation)
465 addPass(createAArch64BranchRelaxation());
464 addPass(&BranchRelaxationPassID);
465
466466 if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH &&
467467 TM->getTargetTriple().isOSBinFormatMachO())
468468 addPass(createAArch64CollectLOHPass());
3737 AArch64AddressTypePromotion.cpp
3838 AArch64AdvSIMDScalarPass.cpp
3939 AArch64AsmPrinter.cpp
40 AArch64BranchRelaxation.cpp
4140 AArch64CleanupLocalDynamicTLSPass.cpp
4241 AArch64CollectLOH.cpp
4342 AArch64ConditionalCompares.cpp
None # RUN: llc -mtriple=aarch64-unknown -run-pass aarch64-branch-relax -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
0 # RUN: llc -mtriple=aarch64-unknown -run-pass branch-relaxation -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
11 --- |
22 ; ModuleID = '/tmp/test.ll'
33 source_filename = "test.ll"
None # RUN: llc -mtriple=aarch64-unknown -run-pass aarch64-branch-relax -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
0 # RUN: llc -mtriple=aarch64-unknown -run-pass branch-relaxation -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
11 --- |
22 ; ModuleID = 'test.ll'
33 source_filename = "test.ll"
None # RUN: llc -mtriple=aarch64-unknown -run-pass aarch64-branch-relax -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
0 # RUN: llc -mtriple=aarch64-unknown -run-pass branch-relaxation -aarch64-tbz-offset-bits=4 %s -o - | FileCheck %s
11 --- |
22 ; ModuleID = 'test.ll'
33 source_filename = "test.ll"