llvm.org GIT mirror llvm / c5d4530
[Machine Combiner] Refactor machine reassociation code to be target-independent. No functional change intended. Patch by Haicheng Wu <haicheng@codeaurora.org>! http://reviews.llvm.org/D12887 PR24522 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@248164 91177308-0d34-0410-b5e6-96231b3b80d8 Chad Rosier 4 years ago
9 changed file(s) with 297 addition(s) and 518 deletion(s). Raw diff Collapse all Expand all
2121 ///
2222 namespace MachineCombinerPattern {
2323 // Forward declaration
24 enum MC_PATTERN : int;
24 enum MC_PATTERN : int {
25 // These are commutative variants for reassociating a computation chain. See
26 // the comments before getMachineCombinerPatterns() in TargetInstrInfo.cpp.
27 MC_REASSOC_AX_BY = 0,
28 MC_REASSOC_AX_YB = 1,
29 MC_REASSOC_XA_BY = 2,
30 MC_REASSOC_XA_YB = 3,
31
32 /// Enumeration of instruction pattern supported by AArch64 machine combiner
33 MC_NONE,
34 MC_MULADDW_OP1,
35 MC_MULADDW_OP2,
36 MC_MULSUBW_OP1,
37 MC_MULSUBW_OP2,
38 MC_MULADDWI_OP1,
39 MC_MULSUBWI_OP1,
40 MC_MULADDX_OP1,
41 MC_MULADDX_OP2,
42 MC_MULSUBX_OP1,
43 MC_MULSUBX_OP2,
44 MC_MULADDXI_OP1,
45 MC_MULSUBXI_OP1
46 };
2547 } // end namespace MachineCombinerPattern
2648 } // end namespace llvm
2749
726726 /// \param Pattern - Vector of possible combination patterns
727727 virtual bool getMachineCombinerPatterns(
728728 MachineInstr &Root,
729 SmallVectorImpl &Pattern) const {
730 return false;
731 }
729 SmallVectorImpl &Patterns) const;
730
731 /// Return true if the input \P Inst is part of a chain of dependent ops
732 /// that are suitable for reassociation, otherwise return false.
733 /// If the instruction's operands must be commuted to have a previous
734 /// instruction of the same type define the first source operand, \P Commuted
735 /// will be set to true.
736 bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) const;
737
738 /// Return true when \P Inst is both associative and commutative.
739 virtual bool isAssociativeAndCommutative(const MachineInstr &Inst) const {
740 return false;
741 }
742
743 /// Return true when \P Inst has reassociable operands in the same \P MBB.
744 virtual bool hasReassociableOperands(const MachineInstr &Inst,
745 const MachineBasicBlock *MBB) const;
746
747 /// Return true when \P Inst has reassociable sibling.
748 bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const;
732749
733750 /// When getMachineCombinerPatterns() finds patterns, this function generates
734751 /// the instructions that could replace the original code sequence. The client
744761 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
745762 SmallVectorImpl &InsInstrs,
746763 SmallVectorImpl &DelInstrs,
747 DenseMap &InstrIdxForVirtReg) const {
764 DenseMap &InstrIdxForVirtReg) const;
765
766 /// Attempt to reassociate \P Root and \P Prev according to \P Pattern to
767 /// reduce critical path length.
768 void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
769 MachineCombinerPattern::MC_PATTERN Pattern,
770 SmallVectorImpl &InsInstrs,
771 SmallVectorImpl &DelInstrs,
772 DenseMap &InstrIdxForVirtReg) const;
773
774 /// This is an architecture-specific helper function of reassociateOps.
775 /// Set special operand attributes for new instructions after reassociation.
776 virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2,
777 MachineInstr &NewMI1,
778 MachineInstr &NewMI2) const {
748779 return;
749 }
780 };
750781
751782 /// Return true when a target supports MachineCombiner.
752783 virtual bool useMachineCombiner() const { return false; }
510510 return --Pos;
511511 }
512512
513 bool TargetInstrInfo::hasReassociableOperands(
514 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
515 const MachineOperand &Op1 = Inst.getOperand(1);
516 const MachineOperand &Op2 = Inst.getOperand(2);
517 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
518
519 // We need virtual register definitions for the operands that we will
520 // reassociate.
521 MachineInstr *MI1 = nullptr;
522 MachineInstr *MI2 = nullptr;
523 if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
524 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
525 if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
526 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
527
528 // And they need to be in the trace (otherwise, they won't have a depth).
529 if (MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB)
530 return true;
531
532 return false;
533 }
534
535 bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
536 bool &Commuted) const {
537 const MachineBasicBlock *MBB = Inst.getParent();
538 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
539 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
540 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
541 unsigned AssocOpcode = Inst.getOpcode();
542
543 // If only one operand has the same opcode and it's the second source operand,
544 // the operands must be commuted.
545 Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
546 if (Commuted)
547 std::swap(MI1, MI2);
548
549 // 1. The previous instruction must be the same type as Inst.
550 // 2. The previous instruction must have virtual register definitions for its
551 // operands in the same basic block as Inst.
552 // 3. The previous instruction's result must only be used by Inst.
553 if (MI1->getOpcode() == AssocOpcode && hasReassociableOperands(*MI1, MBB) &&
554 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()))
555 return true;
556
557 return false;
558 }
559
560 // 1. The operation must be associative and commutative.
561 // 2. The instruction must have virtual register definitions for its
562 // operands in the same basic block.
563 // 3. The instruction must have a reassociable sibling.
564 bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
565 bool &Commuted) const {
566 if (isAssociativeAndCommutative(Inst) &&
567 hasReassociableOperands(Inst, Inst.getParent()) &&
568 hasReassociableSibling(Inst, Commuted))
569 return true;
570
571 return false;
572 }
573
574 // The concept of the reassociation pass is that these operations can benefit
575 // from this kind of transformation:
576 //
577 // A = ? op ?
578 // B = A op X (Prev)
579 // C = B op Y (Root)
580 // -->
581 // A = ? op ?
582 // B = X op Y
583 // C = A op B
584 //
585 // breaking the dependency between A and B, allowing them to be executed in
586 // parallel (or back-to-back in a pipeline) instead of depending on each other.
587
588 // FIXME: This has the potential to be expensive (compile time) while not
589 // improving the code at all. Some ways to limit the overhead:
590 // 1. Track successful transforms; bail out if hit rate gets too low.
591 // 2. Only enable at -O3 or some other non-default optimization level.
592 // 3. Pre-screen pattern candidates here: if an operand of the previous
593 // instruction is known to not increase the critical path, then don't match
594 // that pattern.
595 bool TargetInstrInfo::getMachineCombinerPatterns(
596 MachineInstr &Root,
597 SmallVectorImpl &Patterns) const {
598
599 bool Commute;
600 if (isReassociationCandidate(Root, Commute)) {
601 // We found a sequence of instructions that may be suitable for a
602 // reassociation of operands to increase ILP. Specify each commutation
603 // possibility for the Prev instruction in the sequence and let the
604 // machine combiner decide if changing the operands is worthwhile.
605 if (Commute) {
606 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_YB);
607 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_YB);
608 } else {
609 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_BY);
610 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_BY);
611 }
612 return true;
613 }
614
615 return false;
616 }
617
618 /// Attempt the reassociation transformation to reduce critical path length.
619 /// See the above comments before getMachineCombinerPatterns().
620 void TargetInstrInfo::reassociateOps(
621 MachineInstr &Root, MachineInstr &Prev,
622 MachineCombinerPattern::MC_PATTERN Pattern,
623 SmallVectorImpl &InsInstrs,
624 SmallVectorImpl &DelInstrs,
625 DenseMap &InstrIdxForVirtReg) const {
626 MachineFunction *MF = Root.getParent()->getParent();
627 MachineRegisterInfo &MRI = MF->getRegInfo();
628 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
629 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
630 const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
631
632 // This array encodes the operand index for each parameter because the
633 // operands may be commuted. Each row corresponds to a pattern value,
634 // and each column specifies the index of A, B, X, Y.
635 unsigned OpIdx[4][4] = {
636 { 1, 1, 2, 2 },
637 { 1, 2, 2, 1 },
638 { 2, 1, 1, 2 },
639 { 2, 2, 1, 1 }
640 };
641
642 MachineOperand &OpA = Prev.getOperand(OpIdx[Pattern][0]);
643 MachineOperand &OpB = Root.getOperand(OpIdx[Pattern][1]);
644 MachineOperand &OpX = Prev.getOperand(OpIdx[Pattern][2]);
645 MachineOperand &OpY = Root.getOperand(OpIdx[Pattern][3]);
646 MachineOperand &OpC = Root.getOperand(0);
647
648 unsigned RegA = OpA.getReg();
649 unsigned RegB = OpB.getReg();
650 unsigned RegX = OpX.getReg();
651 unsigned RegY = OpY.getReg();
652 unsigned RegC = OpC.getReg();
653
654 if (TargetRegisterInfo::isVirtualRegister(RegA))
655 MRI.constrainRegClass(RegA, RC);
656 if (TargetRegisterInfo::isVirtualRegister(RegB))
657 MRI.constrainRegClass(RegB, RC);
658 if (TargetRegisterInfo::isVirtualRegister(RegX))
659 MRI.constrainRegClass(RegX, RC);
660 if (TargetRegisterInfo::isVirtualRegister(RegY))
661 MRI.constrainRegClass(RegY, RC);
662 if (TargetRegisterInfo::isVirtualRegister(RegC))
663 MRI.constrainRegClass(RegC, RC);
664
665 // Create a new virtual register for the result of (X op Y) instead of
666 // recycling RegB because the MachineCombiner's computation of the critical
667 // path requires a new register definition rather than an existing one.
668 unsigned NewVR = MRI.createVirtualRegister(RC);
669 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
670
671 unsigned Opcode = Root.getOpcode();
672 bool KillA = OpA.isKill();
673 bool KillX = OpX.isKill();
674 bool KillY = OpY.isKill();
675
676 // Create new instructions for insertion.
677 MachineInstrBuilder MIB1 =
678 BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
679 .addReg(RegX, getKillRegState(KillX))
680 .addReg(RegY, getKillRegState(KillY));
681 MachineInstrBuilder MIB2 =
682 BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
683 .addReg(RegA, getKillRegState(KillA))
684 .addReg(NewVR, getKillRegState(true));
685
686 setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
687
688 // Record new instructions for insertion and old instructions for deletion.
689 InsInstrs.push_back(MIB1);
690 InsInstrs.push_back(MIB2);
691 DelInstrs.push_back(&Prev);
692 DelInstrs.push_back(&Root);
693 }
694
695 void TargetInstrInfo::genAlternativeCodeSequence(
696 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
697 SmallVectorImpl &InsInstrs,
698 SmallVectorImpl &DelInstrs,
699 DenseMap &InstIdxForVirtReg) const {
700 MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo();
701
702 // Select the previous instruction in the sequence based on the input pattern.
703 MachineInstr *Prev = nullptr;
704 switch (Pattern) {
705 case MachineCombinerPattern::MC_REASSOC_AX_BY:
706 case MachineCombinerPattern::MC_REASSOC_XA_BY:
707 Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
708 break;
709 case MachineCombinerPattern::MC_REASSOC_AX_YB:
710 case MachineCombinerPattern::MC_REASSOC_XA_YB:
711 Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
712 break;
713 default:
714 break;
715 }
716
717 assert(Prev && "Unknown pattern for machine combiner");
718
719 reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
720 return;
721 }
722
513723 /// foldMemoryOperand - Same as the previous version except it allows folding
514724 /// of any load and store from / to any address, not just from a specific
515725 /// stack slot.
1111 //===----------------------------------------------------------------------===//
1212
1313 #include "AArch64InstrInfo.h"
14 #include "AArch64MachineCombinerPattern.h"
1514 #include "AArch64Subtarget.h"
1615 #include "MCTargetDesc/AArch64AddressingModes.h"
1716 #include "llvm/CodeGen/MachineFrameInfo.h"
+0
-42
lib/Target/AArch64/AArch64MachineCombinerPattern.h less more
None //===- AArch64MachineCombinerPattern.h -===//
1 //===- AArch64 instruction pattern supported by combiner -===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines instruction pattern supported by combiner
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64MACHINECOMBINERPATTERN_H
15 #define LLVM_LIB_TARGET_AARCH64_AARCH64MACHINECOMBINERPATTERN_H
16
17 namespace llvm {
18
19 /// Enumeration of instruction pattern supported by machine combiner
20 ///
21 ///
22 namespace MachineCombinerPattern {
23 enum MC_PATTERN : int {
24 MC_NONE = 0,
25 MC_MULADDW_OP1 = 1,
26 MC_MULADDW_OP2 = 2,
27 MC_MULSUBW_OP1 = 3,
28 MC_MULSUBW_OP2 = 4,
29 MC_MULADDWI_OP1 = 5,
30 MC_MULSUBWI_OP1 = 6,
31 MC_MULADDX_OP1 = 7,
32 MC_MULADDX_OP2 = 8,
33 MC_MULSUBX_OP1 = 9,
34 MC_MULSUBX_OP2 = 10,
35 MC_MULADDXI_OP1 = 11,
36 MC_MULSUBXI_OP1 = 12
37 };
38 } // end namespace MachineCombinerPattern
39 } // end namespace llvm
40
41 #endif
188188 return Latency;
189189 }
190190
191 static bool hasVirtualRegDefsInBasicBlock(const MachineInstr &Inst,
192 const MachineBasicBlock *MBB) {
193 const MachineOperand &Op1 = Inst.getOperand(1);
194 const MachineOperand &Op2 = Inst.getOperand(2);
195 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
196
197 // We need virtual register definitions.
198 MachineInstr *MI1 = nullptr;
199 MachineInstr *MI2 = nullptr;
200 if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
201 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
202 if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
203 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
204
205 // And they need to be in the trace (otherwise, they won't have a depth).
206 if (MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB)
207 return true;
208
209 return false;
210 }
211
212 static bool hasReassocSibling(const MachineInstr &Inst, bool &Commuted) {
213 const MachineBasicBlock *MBB = Inst.getParent();
214 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
215 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
216 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
217 unsigned AssocOpcode = Inst.getOpcode();
218
219 // If only one operand has the same opcode and it's the second source operand,
220 // the operands must be commuted.
221 Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
222 if (Commuted)
223 std::swap(MI1, MI2);
224
225 // 1. The previous instruction must be the same type as Inst.
226 // 2. The previous instruction must have virtual register definitions for its
227 // operands in the same basic block as Inst.
228 // 3. The previous instruction's result must only be used by Inst.
229 if (MI1->getOpcode() == AssocOpcode &&
230 hasVirtualRegDefsInBasicBlock(*MI1, MBB) &&
231 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()))
232 return true;
233
234 return false;
235 }
236
237191 // This function does not list all associative and commutative operations, but
238192 // only those worth feeding through the machine combiner in an attempt to
239193 // reduce the critical path. Mostly, this means floating-point operations,
240194 // because they have high latencies (compared to other operations, such and
241195 // and/or, which are also associative and commutative, but have low latencies).
242 //
243 // The concept is that these operations can benefit from this kind of
244 // transformation:
245 //
246 // A = ? op ?
247 // B = A op X
248 // C = B op Y
249 // -->
250 // A = ? op ?
251 // B = X op Y
252 // C = A op B
253 //
254 // breaking the dependency between A and B, allowing them to be executed in
255 // parallel (or back-to-back in a pipeline) instead of depending on each other.
256 static bool isAssociativeAndCommutative(unsigned Opcode) {
257 switch (Opcode) {
196 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
197 switch (Inst.getOpcode()) {
258198 // FP Add:
259199 case PPC::FADD:
260200 case PPC::FADDS:
287227 }
288228 }
289229
290 /// Return true if the input instruction is part of a chain of dependent ops
291 /// that are suitable for reassociation, otherwise return false.
292 /// If the instruction's operands must be commuted to have a previous
293 /// instruction of the same type define the first source operand, Commuted will
294 /// be set to true.
295 static bool isReassocCandidate(const MachineInstr &Inst, bool &Commuted) {
296 // 1. The operation must be associative and commutative.
297 // 2. The instruction must have virtual register definitions for its
298 // operands in the same basic block.
299 // 3. The instruction must have a reassociable sibling.
300 if (isAssociativeAndCommutative(Inst.getOpcode()) &&
301 hasVirtualRegDefsInBasicBlock(Inst, Inst.getParent()) &&
302 hasReassocSibling(Inst, Commuted))
303 return true;
304
305 return false;
306 }
307
308 bool PPCInstrInfo::getMachineCombinerPatterns(MachineInstr &Root,
309 SmallVectorImpl &Patterns) const {
230 bool PPCInstrInfo::getMachineCombinerPatterns(
231 MachineInstr &Root,
232 SmallVectorImpl &Patterns) const {
310233 // Using the machine combiner in this way is potentially expensive, so
311234 // restrict to when aggressive optimizations are desired.
312235 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
316239 if (!Root.getParent()->getParent()->getTarget().Options.UnsafeFPMath)
317240 return false;
318241
319 // Look for this reassociation pattern:
320 // B = A op X (Prev)
321 // C = B op Y (Root)
322
323 // FIXME: We should also match FMA operations here, where we consider the
324 // 'part' of the FMA, either the addition or the multiplication, paired with
325 // an actual addition or multiplication.
326
327 bool Commute;
328 if (isReassocCandidate(Root, Commute)) {
329 // We found a sequence of instructions that may be suitable for a
330 // reassociation of operands to increase ILP. Specify each commutation
331 // possibility for the Prev instruction in the sequence and let the
332 // machine combiner decide if changing the operands is worthwhile.
333 if (Commute) {
334 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_YB);
335 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_YB);
336 } else {
337 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_BY);
338 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_BY);
339 }
340 return true;
341 }
342
343 return false;
344 }
345
346 /// Attempt the following reassociation to reduce critical path length:
347 /// B = A op X (Prev)
348 /// C = B op Y (Root)
349 /// ===>
350 /// B = X op Y
351 /// C = A op B
352 static void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
353 MachineCombinerPattern::MC_PATTERN Pattern,
354 SmallVectorImpl &InsInstrs,
355 SmallVectorImpl &DelInstrs,
356 DenseMap &InstrIdxForVirtReg) {
357 MachineFunction *MF = Root.getParent()->getParent();
358 MachineRegisterInfo &MRI = MF->getRegInfo();
359 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
360 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
361 const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
362
363 // This array encodes the operand index for each parameter because the
364 // operands may be commuted. Each row corresponds to a pattern value,
365 // and each column specifies the index of A, B, X, Y.
366 unsigned OpIdx[4][4] = {
367 { 1, 1, 2, 2 },
368 { 1, 2, 2, 1 },
369 { 2, 1, 1, 2 },
370 { 2, 2, 1, 1 }
371 };
372
373 MachineOperand &OpA = Prev.getOperand(OpIdx[Pattern][0]);
374 MachineOperand &OpB = Root.getOperand(OpIdx[Pattern][1]);
375 MachineOperand &OpX = Prev.getOperand(OpIdx[Pattern][2]);
376 MachineOperand &OpY = Root.getOperand(OpIdx[Pattern][3]);
377 MachineOperand &OpC = Root.getOperand(0);
378
379 unsigned RegA = OpA.getReg();
380 unsigned RegB = OpB.getReg();
381 unsigned RegX = OpX.getReg();
382 unsigned RegY = OpY.getReg();
383 unsigned RegC = OpC.getReg();
384
385 if (TargetRegisterInfo::isVirtualRegister(RegA))
386 MRI.constrainRegClass(RegA, RC);
387 if (TargetRegisterInfo::isVirtualRegister(RegB))
388 MRI.constrainRegClass(RegB, RC);
389 if (TargetRegisterInfo::isVirtualRegister(RegX))
390 MRI.constrainRegClass(RegX, RC);
391 if (TargetRegisterInfo::isVirtualRegister(RegY))
392 MRI.constrainRegClass(RegY, RC);
393 if (TargetRegisterInfo::isVirtualRegister(RegC))
394 MRI.constrainRegClass(RegC, RC);
395
396 // Create a new virtual register for the result of (X op Y) instead of
397 // recycling RegB because the MachineCombiner's computation of the critical
398 // path requires a new register definition rather than an existing one.
399 unsigned NewVR = MRI.createVirtualRegister(RC);
400 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
401
402 unsigned Opcode = Root.getOpcode();
403 bool KillA = OpA.isKill();
404 bool KillX = OpX.isKill();
405 bool KillY = OpY.isKill();
406
407 // Create new instructions for insertion.
408 MachineInstrBuilder MIB1 =
409 BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
410 .addReg(RegX, getKillRegState(KillX))
411 .addReg(RegY, getKillRegState(KillY));
412 InsInstrs.push_back(MIB1);
413
414 MachineInstrBuilder MIB2 =
415 BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
416 .addReg(RegA, getKillRegState(KillA))
417 .addReg(NewVR, getKillRegState(true));
418 InsInstrs.push_back(MIB2);
419
420 // Record old instructions for deletion.
421 DelInstrs.push_back(&Prev);
422 DelInstrs.push_back(&Root);
423 }
424
425 void PPCInstrInfo::genAlternativeCodeSequence(
426 MachineInstr &Root,
427 MachineCombinerPattern::MC_PATTERN Pattern,
428 SmallVectorImpl &InsInstrs,
429 SmallVectorImpl &DelInstrs,
430 DenseMap &InstIdxForVirtReg) const {
431 MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo();
432
433 // Select the previous instruction in the sequence based on the input pattern.
434 MachineInstr *Prev = nullptr;
435 switch (Pattern) {
436 case MachineCombinerPattern::MC_REASSOC_AX_BY:
437 case MachineCombinerPattern::MC_REASSOC_XA_BY:
438 Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
439 break;
440 case MachineCombinerPattern::MC_REASSOC_AX_YB:
441 case MachineCombinerPattern::MC_REASSOC_XA_YB:
442 Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
443 }
444 assert(Prev && "Unknown pattern for machine combiner");
445
446 reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
447 return;
242 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
448243 }
449244
450245 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
6262 };
6363 } // end namespace PPCII
6464
65 namespace MachineCombinerPattern {
66 enum MC_PATTERN : int {
67 // These are commutative variants for reassociating a computation chain
68 // of the form:
69 // B = A op X (Prev)
70 // C = B op Y (Root)
71 MC_REASSOC_AX_BY = 0,
72 MC_REASSOC_AX_YB = 1,
73 MC_REASSOC_XA_BY = 2,
74 MC_REASSOC_XA_YB = 3,
75 };
76 } // end namespace MachineCombinerPattern
77
7865 class PPCSubtarget;
7966 class PPCInstrInfo : public PPCGenInstrInfo {
8067 PPCSubtarget &Subtarget;
134121 bool useMachineCombiner() const override {
135122 return true;
136123 }
137
124
138125 /// Return true when there is potentially a faster code sequence
139126 /// for an instruction chain ending in . All potential patterns are
140127 /// output in the array.
141128 bool getMachineCombinerPatterns(
142129 MachineInstr &Root,
143130 SmallVectorImpl &P) const override;
144
145 /// When getMachineCombinerPatterns() finds a pattern, this function generates
146 /// the instructions that could replace the original code sequence.
147 void genAlternativeCodeSequence(
148 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
149 SmallVectorImpl &InsInstrs,
150 SmallVectorImpl &DelInstrs,
151 DenseMap &InstrIdxForVirtReg) const override;
131
132 bool isAssociativeAndCommutative(const MachineInstr &Inst) const override;
152133
153134 bool isCoalescableExtInstr(const MachineInstr &MI,
154135 unsigned &SrcReg, unsigned &DstReg,
63276327 return isHighLatencyDef(DefMI->getOpcode());
63286328 }
63296329
6330 static bool hasReassociableOperands(const MachineInstr &Inst,
6331 const MachineBasicBlock *MBB) {
6330 bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst,
6331 const MachineBasicBlock *MBB) const {
63326332 assert((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) &&
63336333 "Reassociation needs binary operators");
6334 const MachineOperand &Op1 = Inst.getOperand(1);
6335 const MachineOperand &Op2 = Inst.getOperand(2);
6336 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
63376334
63386335 // Integer binary math/logic instructions have a third source operand:
63396336 // the EFLAGS register. That operand must be both defined here and never
63486345 if (!Inst.getOperand(3).isDead())
63496346 return false;
63506347 }
6351
6352 // We need virtual register definitions for the operands that we will
6353 // reassociate.
6354 MachineInstr *MI1 = nullptr;
6355 MachineInstr *MI2 = nullptr;
6356 if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg()))
6357 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
6358 if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg()))
6359 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
6360
6361 // And they need to be in the trace (otherwise, they won't have a depth).
6362 if (MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB)
6363 return true;
6364
6365 return false;
6366 }
6367
6368 static bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) {
6369 const MachineBasicBlock *MBB = Inst.getParent();
6370 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6371 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg());
6372 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
6373 unsigned AssocOpcode = Inst.getOpcode();
6374
6375 // If only one operand has the same opcode and it's the second source operand,
6376 // the operands must be commuted.
6377 Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode;
6378 if (Commuted)
6379 std::swap(MI1, MI2);
6380
6381 // 1. The previous instruction must be the same type as Inst.
6382 // 2. The previous instruction must have virtual register definitions for its
6383 // operands in the same basic block as Inst.
6384 // 3. The previous instruction's result must only be used by Inst.
6385 if (MI1->getOpcode() == AssocOpcode &&
6386 hasReassociableOperands(*MI1, MBB) &&
6387 MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()))
6388 return true;
6389
6390 return false;
6348
6349 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
63916350 }
63926351
63936352 // TODO: There are many more machine instruction opcodes to match:
63946353 // 1. Other data types (integer, vectors)
63956354 // 2. Other math / logic operations (xor, or)
63966355 // 3. Other forms of the same operation (intrinsics and other variants)
6397 static bool isAssociativeAndCommutative(const MachineInstr &Inst) {
6356 bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const {
63986357 switch (Inst.getOpcode()) {
63996358 case X86::AND8rr:
64006359 case X86::AND16rr:
64676426 }
64686427 }
64696428
6470 /// Return true if the input instruction is part of a chain of dependent ops
6471 /// that are suitable for reassociation, otherwise return false.
6472 /// If the instruction's operands must be commuted to have a previous
6473 /// instruction of the same type define the first source operand, Commuted will
6474 /// be set to true.
6475 static bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) {
6476 // 1. The operation must be associative and commutative.
6477 // 2. The instruction must have virtual register definitions for its
6478 // operands in the same basic block.
6479 // 3. The instruction must have a reassociable sibling.
6480 if (isAssociativeAndCommutative(Inst) &&
6481 hasReassociableOperands(Inst, Inst.getParent()) &&
6482 hasReassociableSibling(Inst, Commuted))
6483 return true;
6484
6485 return false;
6486 }
6487
6488 // FIXME: This has the potential to be expensive (compile time) while not
6489 // improving the code at all. Some ways to limit the overhead:
6490 // 1. Track successful transforms; bail out if hit rate gets too low.
6491 // 2. Only enable at -O3 or some other non-default optimization level.
6492 // 3. Pre-screen pattern candidates here: if an operand of the previous
6493 // instruction is known to not increase the critical path, then don't match
6494 // that pattern.
6495 bool X86InstrInfo::getMachineCombinerPatterns(MachineInstr &Root,
6496 SmallVectorImpl &Patterns) const {
6497 // TODO: There is nothing x86-specific here except the instruction type.
6498 // This logic could be hoisted into the machine combiner pass itself.
6499
6500 // Look for this reassociation pattern:
6501 // B = A op X (Prev)
6502 // C = B op Y (Root)
6503
6504 bool Commute;
6505 if (isReassociationCandidate(Root, Commute)) {
6506 // We found a sequence of instructions that may be suitable for a
6507 // reassociation of operands to increase ILP. Specify each commutation
6508 // possibility for the Prev instruction in the sequence and let the
6509 // machine combiner decide if changing the operands is worthwhile.
6510 if (Commute) {
6511 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_YB);
6512 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_YB);
6513 } else {
6514 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_AX_BY);
6515 Patterns.push_back(MachineCombinerPattern::MC_REASSOC_XA_BY);
6516 }
6517 return true;
6518 }
6519
6520 return false;
6521 }
6522
65236429 /// This is an architecture-specific helper function of reassociateOps.
65246430 /// Set special operand attributes for new instructions after reassociation.
6525 static void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2,
6526 MachineInstr &NewMI1, MachineInstr &NewMI2) {
6431 void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
6432 MachineInstr &OldMI2,
6433 MachineInstr &NewMI1,
6434 MachineInstr &NewMI2) const {
65276435 // Integer instructions define an implicit EFLAGS source register operand as
65286436 // the third source (fourth total) operand.
65296437 if (OldMI1.getNumOperands() != 4 || OldMI2.getNumOperands() != 4)
65316439
65326440 assert(NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 &&
65336441 "Unexpected instruction type for reassociation");
6534
6442
65356443 MachineOperand &OldOp1 = OldMI1.getOperand(3);
65366444 MachineOperand &OldOp2 = OldMI2.getOperand(3);
65376445 MachineOperand &NewOp1 = NewMI1.getOperand(3);
65566464 // be dead in order for reassociation to occur.
65576465 NewOp1.setIsDead();
65586466 NewOp2.setIsDead();
6559 }
6560
6561 /// Attempt the following reassociation to reduce critical path length:
6562 /// B = A op X (Prev)
6563 /// C = B op Y (Root)
6564 /// ===>
6565 /// B = X op Y
6566 /// C = A op B
6567 static void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
6568 MachineCombinerPattern::MC_PATTERN Pattern,
6569 SmallVectorImpl &InsInstrs,
6570 SmallVectorImpl &DelInstrs,
6571 DenseMap &InstrIdxForVirtReg) {
6572 MachineFunction *MF = Root.getParent()->getParent();
6573 MachineRegisterInfo &MRI = MF->getRegInfo();
6574 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
6575 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6576 const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
6577
6578 // This array encodes the operand index for each parameter because the
6579 // operands may be commuted. Each row corresponds to a pattern value,
6580 // and each column specifies the index of A, B, X, Y.
6581 unsigned OpIdx[4][4] = {
6582 { 1, 1, 2, 2 },
6583 { 1, 2, 2, 1 },
6584 { 2, 1, 1, 2 },
6585 { 2, 2, 1, 1 }
6586 };
6587
6588 MachineOperand &OpA = Prev.getOperand(OpIdx[Pattern][0]);
6589 MachineOperand &OpB = Root.getOperand(OpIdx[Pattern][1]);
6590 MachineOperand &OpX = Prev.getOperand(OpIdx[Pattern][2]);
6591 MachineOperand &OpY = Root.getOperand(OpIdx[Pattern][3]);
6592 MachineOperand &OpC = Root.getOperand(0);
6593
6594 unsigned RegA = OpA.getReg();
6595 unsigned RegB = OpB.getReg();
6596 unsigned RegX = OpX.getReg();
6597 unsigned RegY = OpY.getReg();
6598 unsigned RegC = OpC.getReg();
6599
6600 if (TargetRegisterInfo::isVirtualRegister(RegA))
6601 MRI.constrainRegClass(RegA, RC);
6602 if (TargetRegisterInfo::isVirtualRegister(RegB))
6603 MRI.constrainRegClass(RegB, RC);
6604 if (TargetRegisterInfo::isVirtualRegister(RegX))
6605 MRI.constrainRegClass(RegX, RC);
6606 if (TargetRegisterInfo::isVirtualRegister(RegY))
6607 MRI.constrainRegClass(RegY, RC);
6608 if (TargetRegisterInfo::isVirtualRegister(RegC))
6609 MRI.constrainRegClass(RegC, RC);
6610
6611 // Create a new virtual register for the result of (X op Y) instead of
6612 // recycling RegB because the MachineCombiner's computation of the critical
6613 // path requires a new register definition rather than an existing one.
6614 unsigned NewVR = MRI.createVirtualRegister(RC);
6615 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
6616
6617 unsigned Opcode = Root.getOpcode();
6618 bool KillA = OpA.isKill();
6619 bool KillX = OpX.isKill();
6620 bool KillY = OpY.isKill();
6621
6622 // Create new instructions for insertion.
6623 MachineInstrBuilder MIB1 =
6624 BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
6625 .addReg(RegX, getKillRegState(KillX))
6626 .addReg(RegY, getKillRegState(KillY));
6627 MachineInstrBuilder MIB2 =
6628 BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
6629 .addReg(RegA, getKillRegState(KillA))
6630 .addReg(NewVR, getKillRegState(true));
6631
6632 setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
6633
6634 // Record new instructions for insertion and old instructions for deletion.
6635 InsInstrs.push_back(MIB1);
6636 InsInstrs.push_back(MIB2);
6637 DelInstrs.push_back(&Prev);
6638 DelInstrs.push_back(&Root);
6639 }
6640
6641 void X86InstrInfo::genAlternativeCodeSequence(
6642 MachineInstr &Root,
6643 MachineCombinerPattern::MC_PATTERN Pattern,
6644 SmallVectorImpl &InsInstrs,
6645 SmallVectorImpl &DelInstrs,
6646 DenseMap &InstIdxForVirtReg) const {
6647 MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo();
6648
6649 // Select the previous instruction in the sequence based on the input pattern.
6650 MachineInstr *Prev = nullptr;
6651 switch (Pattern) {
6652 case MachineCombinerPattern::MC_REASSOC_AX_BY:
6653 case MachineCombinerPattern::MC_REASSOC_XA_BY:
6654 Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
6655 break;
6656 case MachineCombinerPattern::MC_REASSOC_AX_YB:
6657 case MachineCombinerPattern::MC_REASSOC_XA_YB:
6658 Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
6659 }
6660 assert(Prev && "Unknown pattern for machine combiner");
6661
6662 reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
6663 return;
66646467 }
66656468
66666469 std::pair
2424 namespace llvm {
2525 class X86RegisterInfo;
2626 class X86Subtarget;
27
28 namespace MachineCombinerPattern {
29 enum MC_PATTERN : int {
30 // These are commutative variants for reassociating a computation chain
31 // of the form:
32 // B = A op X (Prev)
33 // C = B op Y (Root)
34 MC_REASSOC_AX_BY = 0,
35 MC_REASSOC_AX_YB = 1,
36 MC_REASSOC_XA_BY = 2,
37 MC_REASSOC_XA_YB = 3,
38 };
39 } // end namespace MachineCombinerPattern
4027
4128 namespace X86 {
4229 // X86 specific condition code. These correspond to X86_*_COND in
450437 const MachineInstr *DefMI, unsigned DefIdx,
451438 const MachineInstr *UseMI,
452439 unsigned UseIdx) const override;
453
454440
455441 bool useMachineCombiner() const override {
456442 return true;
457443 }
458
459 /// Return true when there is potentially a faster code sequence
460 /// for an instruction chain ending in . All potential patterns are
461 /// output in the array.
462 bool getMachineCombinerPatterns(
463 MachineInstr &Root,
464 SmallVectorImpl &P) const override;
465
466 /// When getMachineCombinerPatterns() finds a pattern, this function generates
467 /// the instructions that could replace the original code sequence.
468 void genAlternativeCodeSequence(
469 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN P,
470 SmallVectorImpl &InsInstrs,
471 SmallVectorImpl &DelInstrs,
472 DenseMap &InstrIdxForVirtReg) const override;
444
445 bool isAssociativeAndCommutative(const MachineInstr &Inst) const override;
446
447 bool hasReassociableOperands(const MachineInstr &Inst,
448 const MachineBasicBlock *MBB) const override;
449
450 void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2,
451 MachineInstr &NewMI1,
452 MachineInstr &NewMI2) const override;
473453
474454 /// analyzeCompare - For a comparison instruction, return the source registers
475455 /// in SrcReg and SrcReg2 if having two register operands, and the value it