llvm.org GIT mirror llvm / d6ca3f0
Extract LaneBitmask into a separate type Specifically avoid implicit conversions from/to integral types to avoid potential errors when changing the underlying type. For example, a typical initialization of a "full" mask was "LaneMask = ~0u", which would result in a value of 0x00000000FFFFFFFF if the type was extended to uint64_t. Differential Revision: https://reviews.llvm.org/D27454 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@289820 91177308-0d34-0410-b5e6-96231b3b80d8 Krzysztof Parzyszek 3 years ago
40 changed file(s) with 422 addition(s) and 327 deletion(s). Raw diff Collapse all Expand all
458458 void repairOldRegInRange(MachineBasicBlock::iterator Begin,
459459 MachineBasicBlock::iterator End,
460460 const SlotIndex endIdx, LiveRange &LR,
461 unsigned Reg, LaneBitmask LaneMask = ~0u);
461 unsigned Reg,
462 LaneBitmask LaneMask = LaneBitmask::getAll());
462463
463464 class HMEditor;
464465 };
1818 #include "llvm/CodeGen/MachineInstrBundleIterator.h"
1919 #include "llvm/CodeGen/MachineInstr.h"
2020 #include "llvm/Support/BranchProbability.h"
21 #include "llvm/MC/LaneBitmask.h"
2122 #include "llvm/MC/MCRegisterInfo.h"
2223 #include "llvm/Support/DataTypes.h"
2324 #include
3334 class StringRef;
3435 class raw_ostream;
3536 class MachineBranchProbabilityInfo;
36
37 // Forward declaration to avoid circular include problem with TargetRegisterInfo
38 typedef unsigned LaneBitmask;
3937
4038 template <> struct ilist_traits {
4139 private:
277275 /// Adds the specified register as a live in. Note that it is an error to add
278276 /// the same register to the same set more than once unless the intention is
279277 /// to call sortUniqueLiveIns after all registers are added.
280 void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask = ~0u) {
278 void addLiveIn(MCPhysReg PhysReg,
279 LaneBitmask LaneMask = LaneBitmask::getAll()) {
281280 LiveIns.push_back(RegisterMaskPair(PhysReg, LaneMask));
282281 }
283282 void addLiveIn(const RegisterMaskPair &RegMaskPair) {
295294 unsigned addLiveIn(MCPhysReg PhysReg, const TargetRegisterClass *RC);
296295
297296 /// Remove the specified register from the live in set.
298 void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask = ~0u);
297 void removeLiveIn(MCPhysReg Reg,
298 LaneBitmask LaneMask = LaneBitmask::getAll());
299299
300300 /// Return true if the specified register is in the live in set.
301 bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask = ~0u) const;
301 bool isLiveIn(MCPhysReg Reg,
302 LaneBitmask LaneMask = LaneBitmask::getAll()) const;
302303
303304 // Iteration support for live in sets. These sets are kept in sorted
304305 // order by their register number.
277277 unsigned SparseIndex = getSparseIndexFromReg(Reg);
278278 RegSet::const_iterator I = Regs.find(SparseIndex);
279279 if (I == Regs.end())
280 return 0;
280 return LaneBitmask::getNone();
281281 return I->LaneMask;
282282 }
283283
287287 unsigned SparseIndex = getSparseIndexFromReg(Pair.RegUnit);
288288 auto InsertRes = Regs.insert(IndexMaskPair(SparseIndex, Pair.LaneMask));
289289 if (!InsertRes.second) {
290 unsigned PrevMask = InsertRes.first->LaneMask;
290 LaneBitmask PrevMask = InsertRes.first->LaneMask;
291291 InsertRes.first->LaneMask |= Pair.LaneMask;
292292 return PrevMask;
293293 }
294 return 0;
294 return LaneBitmask::getNone();
295295 }
296296
297297 /// Clears the \p Pair.LaneMask lanes of \p Pair.Reg (mark them as dead).
300300 unsigned SparseIndex = getSparseIndexFromReg(Pair.RegUnit);
301301 RegSet::iterator I = Regs.find(SparseIndex);
302302 if (I == Regs.end())
303 return 0;
304 unsigned PrevMask = I->LaneMask;
303 return LaneBitmask::getNone();
304 LaneBitmask PrevMask = I->LaneMask;
305305 I->LaneMask &= ~Pair.LaneMask;
306306 return PrevMask;
307307 }
314314 void appendTo(ContainerT &To) const {
315315 for (const IndexMaskPair &P : Regs) {
316316 unsigned Reg = getRegFromSparseIndex(P.Index);
317 if (P.LaneMask != 0)
317 if (!P.LaneMask.none())
318318 To.push_back(RegisterMaskPair(Reg, P.LaneMask));
319319 }
320320 }
163163 }
164164
165165 /// Tell the scavenger a register is used.
166 void setRegUsed(unsigned Reg, LaneBitmask LaneMask = ~0u);
166 void setRegUsed(unsigned Reg, LaneBitmask LaneMask = LaneBitmask::getAll());
167167 private:
168168 /// Returns true if a register is reserved. It is never "unused".
169169 bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); }
0 //===-- llvm/MC/LaneBitmask.h -----------------------------------*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// A common definition of LaneBitmask for use in TableGen and CodeGen.
11 ///
12 /// A lane mask is a bitmask representing the covering of a register with
13 /// sub-registers.
14 ///
15 /// This is typically used to track liveness at sub-register granularity.
16 /// Lane masks for sub-register indices are similar to register units for
17 /// physical registers. The individual bits in a lane mask can't be assigned
18 /// any specific meaning. They can be used to check if two sub-register
19 /// indices overlap.
20 ///
21 /// Iff the target has a register such that:
22 ///
23 /// getSubReg(Reg, A) overlaps getSubReg(Reg, B)
24 ///
25 /// then:
26 ///
27 /// (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0
28
29 #ifndef LLVM_MC_LANEBITMASK_H
30 #define LLVM_MC_LANEBITMASK_H
31
32 #include "llvm/Support/Format.h"
33 #include "llvm/Support/Printable.h"
34 #include "llvm/Support/raw_ostream.h"
35
36 namespace llvm {
37 struct LaneBitmask {
38 // When changing the underlying type, change the format string as well.
39 typedef unsigned Type;
40 enum : unsigned { BitWidth = 8*sizeof(Type) };
41 constexpr static const char *const FormatStr = "%08X";
42
43 constexpr LaneBitmask() = default;
44 explicit constexpr LaneBitmask(Type V) : Mask(V) {}
45
46 constexpr bool operator== (LaneBitmask M) const { return Mask == M.Mask; }
47 constexpr bool operator!= (LaneBitmask M) const { return Mask != M.Mask; }
48 constexpr bool operator< (LaneBitmask M) const { return Mask < M.Mask; }
49 constexpr bool none() const { return Mask == 0; }
50 constexpr bool all() const { return ~Mask == 0; }
51
52 constexpr LaneBitmask operator~() const {
53 return LaneBitmask(~Mask);
54 }
55 constexpr LaneBitmask operator|(LaneBitmask M) const {
56 return LaneBitmask(Mask | M.Mask);
57 }
58 constexpr LaneBitmask operator&(LaneBitmask M) const {
59 return LaneBitmask(Mask & M.Mask);
60 }
61 LaneBitmask &operator|=(LaneBitmask M) {
62 Mask |= M.Mask;
63 return *this;
64 }
65 LaneBitmask &operator&=(LaneBitmask M) {
66 Mask &= M.Mask;
67 return *this;
68 }
69
70 constexpr Type getAsInteger() const { return Mask; }
71
72 static LaneBitmask getNone() { return LaneBitmask(0); }
73 static LaneBitmask getAll() { return ~LaneBitmask(0); }
74
75 private:
76 Type Mask = 0;
77 };
78
79 /// Create Printable object to print LaneBitmasks on a \ref raw_ostream.
80 static LLVM_ATTRIBUTE_UNUSED Printable PrintLaneMask(LaneBitmask LaneMask) {
81 return Printable([LaneMask](raw_ostream &OS) {
82 OS << format(LaneBitmask::FormatStr, LaneMask.getAsInteger());
83 });
84 }
85 }
86
87 #endif // LLVM_MC_LANEBITMASK_H
1616 #define LLVM_MC_MCREGISTERINFO_H
1717
1818 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/MC/LaneBitmask.h"
1920 #include "llvm/Support/ErrorHandling.h"
2021 #include
2122
160161 unsigned NumRegUnits; // Number of regunits.
161162 const MCPhysReg (*RegUnitRoots)[2]; // Pointer to regunit root table.
162163 const MCPhysReg *DiffLists; // Pointer to the difflists array
163 const unsigned *RegUnitMaskSequences; // Pointer to lane mask sequences
164 const LaneBitmask *RegUnitMaskSequences; // Pointer to lane mask sequences
164165 // for register units.
165166 const char *RegStrings; // Pointer to the string table.
166167 const char *RegClassStrings; // Pointer to the class strings.
247248 const MCPhysReg (*RURoots)[2],
248249 unsigned NRU,
249250 const MCPhysReg *DL,
250 const unsigned *RUMS,
251 const LaneBitmask *RUMS,
251252 const char *Strings,
252253 const char *ClassStrings,
253254 const uint16_t *SubIndices,
583584 /// numerical order.
584585 class MCRegUnitMaskIterator {
585586 MCRegUnitIterator RUIter;
586 const unsigned *MaskListIter;
587 const LaneBitmask *MaskListIter;
587588 public:
588589 MCRegUnitMaskIterator() {}
589590 /// Constructs an iterator that traverses the register units and their
595596 }
596597
597598 /// Returns a (RegUnit, LaneMask) pair.
598 std::pairunsigned> operator*() const {
599 std::pairLaneBitmask> operator*() const {
599600 return std::make_pair(*RUIter, *MaskListIter);
600601 }
601602
3434 class VirtRegMap;
3535 class raw_ostream;
3636 class LiveRegMatrix;
37
38 /// A bitmask representing the covering of a register with sub-registers.
39 ///
40 /// This is typically used to track liveness at sub-register granularity.
41 /// Lane masks for sub-register indices are similar to register units for
42 /// physical registers. The individual bits in a lane mask can't be assigned
43 /// any specific meaning. They can be used to check if two sub-register
44 /// indices overlap.
45 ///
46 /// Iff the target has a register such that:
47 ///
48 /// getSubReg(Reg, A) overlaps getSubReg(Reg, B)
49 ///
50 /// then:
51 ///
52 /// (getSubRegIndexLaneMask(A) & getSubRegIndexLaneMask(B)) != 0
53 typedef unsigned LaneBitmask;
5437
5538 class TargetRegisterClass {
5639 public:
268251 const LaneBitmask *SubRegIndexLaneMasks;
269252
270253 regclass_iterator RegClassBegin, RegClassEnd; // List of regclasses
271 unsigned CoveringLanes;
254 LaneBitmask CoveringLanes;
272255
273256 protected:
274257 TargetRegisterInfo(const TargetRegisterInfoDesc *ID,
276259 regclass_iterator RegClassEnd,
277260 const char *const *SRINames,
278261 const LaneBitmask *SRILaneMasks,
279 unsigned CoveringLanes);
262 LaneBitmask CoveringLanes);
280263 virtual ~TargetRegisterInfo();
281264 public:
282265
11401123 /// registers on a \ref raw_ostream.
11411124 Printable PrintVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *TRI);
11421125
1143 /// Create Printable object to print LaneBitmasks on a \ref raw_ostream.
1144 Printable PrintLaneMask(LaneBitmask LaneMask);
1145
11461126 } // End llvm namespace
11471127
11481128 #endif
209209 VRegInfo &MORegInfo = VRegInfos[MORegIdx];
210210 LaneBitmask PrevUsedLanes = MORegInfo.UsedLanes;
211211 // Any change at all?
212 if ((UsedLanes & ~PrevUsedLanes) == 0)
212 if ((UsedLanes & ~PrevUsedLanes).none())
213213 return;
214214
215215 // Set UsedLanes and remember instruction for further propagation.
302302 VRegInfo &RegInfo = VRegInfos[DefRegIdx];
303303 LaneBitmask PrevDefinedLanes = RegInfo.DefinedLanes;
304304 // Any change at all?
305 if ((DefinedLanes & ~PrevDefinedLanes) == 0)
305 if ((DefinedLanes & ~PrevDefinedLanes).none())
306306 return;
307307
308308 RegInfo.DefinedLanes = PrevDefinedLanes | DefinedLanes;
355355 // Live-In or unused registers have no definition but are considered fully
356356 // defined.
357357 if (!MRI->hasOneDef(Reg))
358 return ~0u;
358 return LaneBitmask::getAll();
359359
360360 const MachineOperand &Def = *MRI->def_begin(Reg);
361361 const MachineInstr &DefMI = *Def.getParent();
367367 PutInWorklist(RegIdx);
368368
369369 if (Def.isDead())
370 return 0;
370 return LaneBitmask::getNone();
371371
372372 // COPY/PHI can copy across unrelated register classes (example: float/int)
373373 // with incompatible subregister structure. Do not include these in the
375375 const TargetRegisterClass *DefRC = MRI->getRegClass(Reg);
376376
377377 // Determine initially DefinedLanes.
378 LaneBitmask DefinedLanes = 0;
378 LaneBitmask DefinedLanes;
379379 for (const MachineOperand &MO : DefMI.uses()) {
380380 if (!MO.isReg() || !MO.readsReg())
381381 continue;
385385
386386 LaneBitmask MODefinedLanes;
387387 if (TargetRegisterInfo::isPhysicalRegister(MOReg)) {
388 MODefinedLanes = ~0u;
388 MODefinedLanes = LaneBitmask::getAll();
389389 } else if (isCrossCopy(*MRI, DefMI, DefRC, MO)) {
390 MODefinedLanes = ~0u;
390 MODefinedLanes = LaneBitmask::getAll();
391391 } else {
392392 assert(TargetRegisterInfo::isVirtualRegister(MOReg));
393393 if (MRI->hasOneDef(MOReg)) {
409409 return DefinedLanes;
410410 }
411411 if (DefMI.isImplicitDef() || Def.isDead())
412 return 0;
412 return LaneBitmask::getNone();
413413
414414 assert(Def.getSubReg() == 0 &&
415415 "Should not have subregister defs in machine SSA phase");
417417 }
418418
419419 LaneBitmask DetectDeadLanes::determineInitialUsedLanes(unsigned Reg) {
420 LaneBitmask UsedLanes = 0;
420 LaneBitmask UsedLanes = LaneBitmask::getNone();
421421 for (const MachineOperand &MO : MRI->use_nodbg_operands(Reg)) {
422422 if (!MO.readsReg())
423423 continue;
461461 const VRegInfo &RegInfo) const {
462462 unsigned SubReg = MO.getSubReg();
463463 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubReg);
464 return (RegInfo.DefinedLanes & RegInfo.UsedLanes & Mask) == 0;
464 return (RegInfo.DefinedLanes & RegInfo.UsedLanes & Mask).none();
465465 }
466466
467467 bool DetectDeadLanes::isUndefInput(const MachineOperand &MO,
481481
482482 const VRegInfo &DefRegInfo = VRegInfos[DefRegIdx];
483483 LaneBitmask UsedLanes = transferUsedLanes(MI, DefRegInfo.UsedLanes, MO);
484 if (UsedLanes != 0)
484 if (!UsedLanes.none())
485485 return false;
486486
487487 unsigned MOReg = MO.getReg();
545545 continue;
546546 unsigned RegIdx = TargetRegisterInfo::virtReg2Index(Reg);
547547 const VRegInfo &RegInfo = VRegInfos[RegIdx];
548 if (MO.isDef() && !MO.isDead() && RegInfo.UsedLanes == 0) {
548 if (MO.isDef() && !MO.isDead() && RegInfo.UsedLanes.none()) {
549549 DEBUG(dbgs() << "Marking operand '" << MO << "' as dead in " << MI);
550550 MO.setIsDead();
551551 }
875875 const SlotIndexes &Indexes) const {
876876 assert(TargetRegisterInfo::isVirtualRegister(reg));
877877 LaneBitmask VRegMask = MRI.getMaxLaneMaskForVReg(reg);
878 assert((VRegMask & LaneMask) != 0);
878 assert(!(VRegMask & LaneMask).none());
879879 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
880880 for (const MachineOperand &MO : MRI.def_operands(reg)) {
881881 if (!MO.isUndef())
884884 assert(SubReg != 0 && "Undef should only be set on subreg defs");
885885 LaneBitmask DefMask = TRI.getSubRegIndexLaneMask(SubReg);
886886 LaneBitmask UndefMask = VRegMask & ~DefMask;
887 if ((UndefMask & LaneMask) != 0) {
887 if (!(UndefMask & LaneMask).none()) {
888888 const MachineInstr &MI = *MO.getParent();
889889 bool EarlyClobber = MO.isEarlyClobber();
890890 SlotIndex Pos = Indexes.getInstructionIndex(MI).getRegSlot(EarlyClobber);
981981 super::verify();
982982
983983 // Make sure SubRanges are fine and LaneMasks are disjunct.
984 LaneBitmask Mask = 0;
985 LaneBitmask MaxMask = MRI != nullptr ? MRI->getMaxLaneMaskForVReg(reg) : ~0u;
984 LaneBitmask Mask;
985 LaneBitmask MaxMask = MRI != nullptr ? MRI->getMaxLaneMaskForVReg(reg)
986 : LaneBitmask::getAll();
986987 for (const SubRange &SR : subranges()) {
987988 // Subrange lanemask should be disjunct to any previous subrange masks.
988 assert((Mask & SR.LaneMask) == 0);
989 assert((Mask & SR.LaneMask).none());
989990 Mask |= SR.LaneMask;
990991
991992 // subrange mask should not contained in maximum lane mask for the vreg.
992 assert((Mask & ~MaxMask) == 0);
993 assert((Mask & ~MaxMask).none());
993994 // empty subranges must be removed.
994995 assert(!SR.empty());
995996
513513 unsigned SubReg = MO.getSubReg();
514514 if (SubReg != 0) {
515515 LaneBitmask LaneMask = TRI->getSubRegIndexLaneMask(SubReg);
516 if ((LaneMask & SR.LaneMask) == 0)
516 if ((LaneMask & SR.LaneMask).none())
517517 continue;
518518 }
519519 // We only need to visit each instruction once.
717717 LaneBitmask DefinedLanesMask;
718718 if (!SRs.empty()) {
719719 // Compute a mask of lanes that are defined.
720 DefinedLanesMask = 0;
720 DefinedLanesMask = LaneBitmask::getNone();
721721 for (auto &SRP : SRs) {
722722 const LiveInterval::SubRange &SR = *SRP.first;
723723 LiveRange::const_iterator &I = SRP.second;
730730 DefinedLanesMask |= SR.LaneMask;
731731 }
732732 } else
733 DefinedLanesMask = ~0u;
733 DefinedLanesMask = LaneBitmask::getAll();
734734
735735 bool IsFullWrite = false;
736736 for (const MachineOperand &MO : MI->operands()) {
739739 if (MO.isUse()) {
740740 // Reading any undefined lanes?
741741 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
742 if ((UseMask & ~DefinedLanesMask) != 0)
742 if (!(UseMask & ~DefinedLanesMask).none())
743743 goto CancelKill;
744744 } else if (MO.getSubReg() == 0) {
745745 // Writing to the full register?
950950 LaneBitmask LaneMask = SubReg ? TRI.getSubRegIndexLaneMask(SubReg)
951951 : MRI.getMaxLaneMaskForVReg(Reg);
952952 for (LiveInterval::SubRange &S : LI.subranges()) {
953 if ((S.LaneMask & LaneMask) == 0)
953 if ((S.LaneMask & LaneMask).none())
954954 continue;
955955 updateRange(S, Reg, S.LaneMask);
956956 }
957957 }
958 updateRange(LI, Reg, 0);
958 updateRange(LI, Reg, LaneBitmask::getNone());
959959 continue;
960960 }
961961
963963 // precomputed live range.
964964 for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
965965 if (LiveRange *LR = getRegUnitLI(*Units))
966 updateRange(*LR, *Units, 0);
966 updateRange(*LR, *Units, LaneBitmask::getNone());
967967 }
968968 if (hasRegMask)
969969 updateRegMaskSlots();
979979 dbgs() << " ";
980980 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
981981 dbgs() << PrintReg(Reg);
982 if (LaneMask != 0)
982 if (!LaneMask.none())
983983 dbgs() << " L" << PrintLaneMask(LaneMask);
984984 } else {
985985 dbgs() << PrintRegUnit(Reg, &TRI);
13131313 if (MO.isUndef())
13141314 continue;
13151315 unsigned SubReg = MO.getSubReg();
1316 if (SubReg != 0 && LaneMask != 0
1317 && (TRI.getSubRegIndexLaneMask(SubReg) & LaneMask) == 0)
1316 if (SubReg != 0 && !LaneMask.none()
1317 && (TRI.getSubRegIndexLaneMask(SubReg) & LaneMask).none())
13181318 continue;
13191319
13201320 const MachineInstr &MI = *MO.getParent();
14211421
14221422 unsigned SubReg = MO.getSubReg();
14231423 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubReg);
1424 if ((Mask & LaneMask) == 0)
1424 if ((Mask & LaneMask).none())
14251425 continue;
14261426
14271427 if (MO.isDef()) {
143143 void LivePhysRegs::addBlockLiveIns(const MachineBasicBlock &MBB) {
144144 for (const auto &LI : MBB.liveins()) {
145145 MCSubRegIndexIterator S(LI.PhysReg, TRI);
146 if (LI.LaneMask == ~0u || (LI.LaneMask != 0 && !S.isValid())) {
146 if (LI.LaneMask.all() || (!LI.LaneMask.none() && !S.isValid())) {
147147 addReg(LI.PhysReg);
148148 continue;
149149 }
150 for (; S.isValid(); ++S)
151 if (LI.LaneMask & TRI->getSubRegIndexLaneMask(S.getSubRegIndex()))
150 for (; S.isValid(); ++S) {
151 unsigned SI = S.getSubRegIndex();
152 if (!(LI.LaneMask & TRI->getSubRegIndexLaneMask(SI)).none())
152153 addReg(S.getSubReg());
154 }
153155 }
154156 }
155157
7878 for (LiveInterval::SubRange &S : LI.subranges()) {
7979 // A Mask for subregs common to the existing subrange and current def.
8080 LaneBitmask Common = S.LaneMask & Mask;
81 if (Common == 0)
81 if (Common.none())
8282 continue;
8383 LiveInterval::SubRange *CommonRange;
8484 // A Mask for subregs covered by the subrange but not the current def.
85 if (LaneBitmask RM = S.LaneMask & ~Mask) {
85 LaneBitmask RM = S.LaneMask & ~Mask;
86 if (!RM.none()) {
8687 // Split the subrange S into two parts: one covered by the current
8788 // def (CommonRange), and the one not affected by it (updated S).
8889 S.LaneMask = RM;
9697 Mask &= ~Common;
9798 }
9899 // Create a new SubRange for subregs we did not cover yet.
99 if (Mask != 0) {
100 if (!Mask.none()) {
100101 LiveInterval::SubRange *NewRange = LI.createSubRange(*Alloc, Mask);
101102 if (MO.isDef())
102103 createDeadDef(*Indexes, *Alloc, *NewRange, MO);
125126 constructMainRangeFromSubranges(LI);
126127 } else {
127128 resetLiveOutMap();
128 extendToUses(LI, Reg, ~0u);
129 extendToUses(LI, Reg, LaneBitmask::getAll());
129130 }
130131 }
131132
142143 }
143144 }
144145 resetLiveOutMap();
145 extendToUses(MainRange, LI.reg, ~0U, &LI);
146 extendToUses(MainRange, LI.reg, LaneBitmask::getAll(), &LI);
146147 }
147148
148149 void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) {
162163 LI->computeSubRangeUndefs(Undefs, Mask, *MRI, *Indexes);
163164
164165 // Visit all operands that read Reg. This may include partial defs.
165 bool IsSubRange = (Mask != ~0U);
166 bool IsSubRange = !Mask.all();
166167 const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
167168 for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
168169 // Clear all kill flags. They will be reinserted after register allocation
182183 if (MO.isDef())
183184 SLM = ~SLM;
184185 // Ignore uses not reading the current (sub)range.
185 if ((SLM & Mask) == 0)
186 if ((SLM & Mask).none())
186187 continue;
187188 }
188189
159159 /// all uses must be jointly dominated by the definitions from @p LR
160160 /// together with definitions of other lanes where @p LR becomes undefined
161161 /// (via operands).
162 /// If @p LR is a main range, the @p LaneMask should be set to ~0.
162 /// If @p LR is a main range, the @p LaneMask should be set to ~0, i.e.
163 /// LaneBitmask::getAll().
163164 void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask,
164165 LiveInterval *LI = nullptr);
165166
214215 /// All uses must be jointly dominated by existing liveness. PHI-defs are
215216 /// inserted as needed to preserve SSA form.
216217 void extendToUses(LiveRange &LR, unsigned PhysReg) {
217 extendToUses(LR, PhysReg, ~0u);
218 extendToUses(LR, PhysReg, LaneBitmask::getAll());
218219 }
219220
220221 /// Calculates liveness for the register specified in live interval @p LI.
235235 unsigned SubReg = MO.getSubReg();
236236 LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubReg);
237237 for (const LiveInterval::SubRange &S : LI.subranges()) {
238 if ((S.LaneMask & LaneMask) != 0 && S.Query(Idx).isKill())
238 if (!(S.LaneMask & LaneMask).none() && S.Query(Idx).isKill())
239239 return true;
240240 }
241241 return false;
7878 unsigned Unit = (*Units).first;
7979 LaneBitmask Mask = (*Units).second;
8080 for (LiveInterval::SubRange &S : VRegInterval.subranges()) {
81 if (S.LaneMask & Mask) {
81 if (!(S.LaneMask & Mask).none()) {
8282 if (Func(Unit, S))
8383 return true;
8484 break;
455455 if (parseNamedRegister(Reg))
456456 return true;
457457 lex();
458 LaneBitmask Mask = ~LaneBitmask(0);
458 LaneBitmask Mask = LaneBitmask::getAll();
459459 if (consumeIfPresent(MIToken::colon)) {
460460 // Parse lane mask.
461461 if (Token.isNot(MIToken::IntegerLiteral) &&
462462 Token.isNot(MIToken::HexLiteral))
463463 return error("expected a lane mask");
464 static_assert(sizeof(LaneBitmask) == sizeof(unsigned), "");
465 if (getUnsigned(Mask))
464 static_assert(sizeof(LaneBitmask::Type) == sizeof(unsigned),
465 "Use correct get-function for lane mask");
466 LaneBitmask::Type V;
467 if (getUnsigned(V))
466468 return error("invalid lane mask value");
469 Mask = LaneBitmask(V);
467470 lex();
468471 }
469472 MBB.addLiveIn(Reg, Mask);
497497 OS << ", ";
498498 First = false;
499499 printReg(LI.PhysReg, OS, TRI);
500 if (LI.LaneMask != ~0u)
500 if (!LI.LaneMask.all())
501501 OS << ":0x" << PrintLaneMask(LI.LaneMask);
502502 }
503503 OS << "\n";
290290 OS << " Live Ins:";
291291 for (const auto &LI : make_range(livein_begin(), livein_end())) {
292292 OS << ' ' << PrintReg(LI.PhysReg, TRI);
293 if (LI.LaneMask != ~0u)
293 if (!LI.LaneMask.all())
294294 OS << ':' << PrintLaneMask(LI.LaneMask);
295295 }
296296 OS << '\n';
341341 return;
342342
343343 I->LaneMask &= ~LaneMask;
344 if (I->LaneMask == 0)
344 if (I->LaneMask.none())
345345 LiveIns.erase(I);
346346 }
347347
348348 bool MachineBasicBlock::isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask) const {
349349 livein_iterator I = find_if(
350350 LiveIns, [Reg](const RegisterMaskPair &LI) { return LI.PhysReg == Reg; });
351 return I != livein_end() && (I->LaneMask & LaneMask) != 0;
351 return I != livein_end() && !(I->LaneMask & LaneMask).none();
352352 }
353353
354354 void MachineBasicBlock::sortUniqueLiveIns() {
17411741 unsigned Reg = MO.getReg();
17421742 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
17431743 if (!Uses.count(Reg))
1744 LiveOutRegs.push_back(RegisterMaskPair(Reg, 0));
1744 LiveOutRegs.push_back(RegisterMaskPair(Reg,
1745 LaneBitmask::getNone()));
17451746 } else if (MRI.isAllocatable(Reg)) {
17461747 for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
17471748 if (!Uses.count(*Units))
1748 LiveOutRegs.push_back(RegisterMaskPair(*Units, 0));
1749 LiveOutRegs.push_back(RegisterMaskPair(*Units,
1750 LaneBitmask::getNone()));
17491751 }
17501752 }
17511753 RPTracker.addLiveRegs(LiveOutRegs);
893893 break;
894894 }
895895 if (UI == VRegUses.end())
896 VRegUses.insert(VReg2SUnit(Reg, 0, &SU));
896 VRegUses.insert(VReg2SUnit(Reg, LaneBitmask::getNone(), &SU));
897897 }
898898 }
899899
10391039 // this fact anymore => decrement pressure.
10401040 // If the register has just become dead then other uses make it come
10411041 // back to life => increment pressure.
1042 bool Decrement = P.LaneMask != 0;
1042 bool Decrement = !P.LaneMask.none();
10431043
10441044 for (const VReg2SUnit &V2SU
10451045 : make_range(VRegUses.find(Reg), VRegUses.end())) {
10581058 );
10591059 }
10601060 } else {
1061 assert(P.LaneMask != 0);
1061 assert(!P.LaneMask.none());
10621062 DEBUG(dbgs() << " LiveReg: " << PrintVRegOrUnit(Reg, TRI) << "\n");
10631063 // This may be called before CurrentBottom has been initialized. However,
10641064 // BotRPTracker must have a valid position. We want the value live into the
228228 void checkLiveness(const MachineOperand *MO, unsigned MONum);
229229 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
230230 SlotIndex UseIdx, const LiveRange &LR, unsigned Reg,
231 LaneBitmask LaneMask = 0);
231 LaneBitmask LaneMask = LaneBitmask::getNone());
232232 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
233233 SlotIndex DefIdx, const LiveRange &LR, unsigned Reg,
234 LaneBitmask LaneMask = 0);
234 LaneBitmask LaneMask = LaneBitmask::getNone());
235235
236236 void markReachable(const MachineBasicBlock *MBB);
237237 void calcRegsPassed();
242242 void verifyLiveIntervals();
243243 void verifyLiveInterval(const LiveInterval&);
244244 void verifyLiveRangeValue(const LiveRange&, const VNInfo*, unsigned,
245 unsigned);
245 LaneBitmask);
246246 void verifyLiveRangeSegment(const LiveRange&,
247247 const LiveRange::const_iterator I, unsigned,
248 unsigned);
249 void verifyLiveRange(const LiveRange&, unsigned, LaneBitmask LaneMask = 0);
248 LaneBitmask);
249 void verifyLiveRange(const LiveRange&, unsigned,
250 LaneBitmask LaneMask = LaneBitmask::getNone());
250251
251252 void verifyStackFrame();
252253
480481 LaneBitmask LaneMask) const {
481482 report_context_liverange(LR);
482483 report_context_vreg_regunit(VRegUnit);
483 if (LaneMask != 0)
484 if (!LaneMask.none())
484485 report_context_lanemask(LaneMask);
485486 }
486487
11601161 LiveQueryResult LRQ = LR.Query(UseIdx);
11611162 // Check if we have a segment at the use, note however that we only need one
11621163 // live subregister range, the others may be dead.
1163 if (!LRQ.valueIn() && LaneMask == 0) {
1164 if (!LRQ.valueIn() && LaneMask.none()) {
11641165 report("No live segment at use", MO, MONum);
11651166 report_context_liverange(LR);
11661167 report_context_vreg_regunit(VRegOrUnit);
11701171 report("Live range continues after kill flag", MO, MONum);
11711172 report_context_liverange(LR);
11721173 report_context_vreg_regunit(VRegOrUnit);
1173 if (LaneMask != 0)
1174 if (!LaneMask.none())
11741175 report_context_lanemask(LaneMask);
11751176 report_context(UseIdx);
11761177 }
11851186 report("Inconsistent valno->def", MO, MONum);
11861187 report_context_liverange(LR);
11871188 report_context_vreg_regunit(VRegOrUnit);
1188 if (LaneMask != 0)
1189 if (!LaneMask.none())
11891190 report_context_lanemask(LaneMask);
11901191 report_context(*VNI);
11911192 report_context(DefIdx);
11941195 report("No live segment at def", MO, MONum);
11951196 report_context_liverange(LR);
11961197 report_context_vreg_regunit(VRegOrUnit);
1197 if (LaneMask != 0)
1198 if (!LaneMask.none())
11981199 report_context_lanemask(LaneMask);
11991200 report_context(DefIdx);
12001201 }
12241225 report("Live range continues after dead def flag", MO, MONum);
12251226 report_context_liverange(LR);
12261227 report_context_vreg_regunit(VRegOrUnit);
1227 if (LaneMask != 0)
1228 if (!LaneMask.none())
12281229 report_context_lanemask(LaneMask);
12291230 }
12301231 }
12721273 LaneBitmask MOMask = SubRegIdx != 0
12731274 ? TRI->getSubRegIndexLaneMask(SubRegIdx)
12741275 : MRI->getMaxLaneMaskForVReg(Reg);
1275 LaneBitmask LiveInMask = 0;
1276 LaneBitmask LiveInMask;
12761277 for (const LiveInterval::SubRange &SR : LI.subranges()) {
1277 if ((MOMask & SR.LaneMask) == 0)
1278 if ((MOMask & SR.LaneMask).none())
12781279 continue;
12791280 checkLivenessAtUse(MO, MONum, UseIdx, SR, Reg, SR.LaneMask);
12801281 LiveQueryResult LRQ = SR.Query(UseIdx);
12821283 LiveInMask |= SR.LaneMask;
12831284 }
12841285 // At least parts of the register has to be live at the use.
1285 if ((LiveInMask & MOMask) == 0) {
1286 if ((LiveInMask & MOMask).none()) {
12861287 report("No live subrange at use", MO, MONum);
12871288 report_context(LI);
12881289 report_context(UseIdx);
13741375 ? TRI->getSubRegIndexLaneMask(SubRegIdx)
13751376 : MRI->getMaxLaneMaskForVReg(Reg);
13761377 for (const LiveInterval::SubRange &SR : LI.subranges()) {
1377 if ((SR.LaneMask & MOMask) == 0)
1378 if ((SR.LaneMask & MOMask).none())
13781379 continue;
13791380 checkLivenessAtDef(MO, MONum, DefIdx, SR, Reg, SR.LaneMask);
13801381 }
16871688 !TRI->hasRegUnit(MOI->getReg(), Reg))
16881689 continue;
16891690 }
1690 if (LaneMask != 0 &&
1691 (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask) == 0)
1691 if (!LaneMask.none() &&
1692 (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())
16921693 continue;
16931694 hasDef = true;
16941695 if (MOI->isEarlyClobber())
18201821 if (!MOI->isReg() || MOI->getReg() != Reg)
18211822 continue;
18221823 unsigned Sub = MOI->getSubReg();
1823 LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : ~0U;
1824 LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub)
1825 : LaneBitmask::getAll();
18241826 if (MOI->isDef()) {
18251827 if (Sub != 0) {
18261828 hasSubRegDef = true;
18321834 if (MOI->isDead())
18331835 hasDeadDef = true;
18341836 }
1835 if (LaneMask != 0 && !(LaneMask & SLM))
1837 if (!LaneMask.none() && (LaneMask & SLM).none())
18361838 continue;
18371839 if (MOI->readsReg())
18381840 hasRead = true;
18411843 // Make sure that the corresponding machine operand for a "dead" live
18421844 // range has the dead flag. We cannot perform this check for subregister
18431845 // liveranges as partially dead values are allowed.
1844 if (LaneMask == 0 && !hasDeadDef) {
1846 if (LaneMask.none() && !hasDeadDef) {
18451847 report("Instruction ending live segment on dead slot has no dead flag",
18461848 MI);
18471849 report_context(LR, Reg, LaneMask);
18511853 if (!hasRead) {
18521854 // When tracking subregister liveness, the main range must start new
18531855 // values on partial register writes, even if there is no read.
1854 if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask != 0 ||
1856 if (!MRI->shouldTrackSubRegLiveness(Reg) || !LaneMask.none() ||
18551857 !hasSubRegDef) {
18561858 report("Instruction ending live segment doesn't read the register",
18571859 MI);
18951897
18961898 // All predecessors must have a live-out value if this is not a
18971899 // subregister liverange.
1898 if (!PVNI && LaneMask == 0) {
1900 if (!PVNI && LaneMask.none()) {
18991901 report("Register not marked live out of predecessor", *PI);
19001902 report_context(LR, Reg, LaneMask);
19011903 report_context(*VNI);
19351937 assert(TargetRegisterInfo::isVirtualRegister(Reg));
19361938 verifyLiveRange(LI, Reg);
19371939
1938 LaneBitmask Mask = 0;
1940 LaneBitmask Mask;
19391941 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
19401942 for (const LiveInterval::SubRange &SR : LI.subranges()) {
1941 if ((Mask & SR.LaneMask) != 0) {
1943 if (!(Mask & SR.LaneMask).none()) {
19421944 report("Lane masks of sub ranges overlap in live interval", MF);
19431945 report_context(LI);
19441946 }
1945 if ((SR.LaneMask & ~MaxMask) != 0) {
1947 if (!(SR.LaneMask & ~MaxMask).none()) {
19461948 report("Subrange lanemask is invalid", MF);
19471949 report_context(LI);
19481950 }
18321832 // sub-register we are tracking.
18331833 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
18341834 if (!TRI ||
1835 (TRI->getSubRegIndexLaneMask(DefSubReg) &
1836 TRI->getSubRegIndexLaneMask(InsertedReg.SubIdx)) != 0)
1835 !(TRI->getSubRegIndexLaneMask(DefSubReg) &
1836 TRI->getSubRegIndexLaneMask(InsertedReg.SubIdx)).none())
18371837 return ValueTrackerResult();
18381838 // At this point, the value is available in v0 via the same subreg
18391839 // we used for Def.
814814 for (LiveInterval::SubRange &SB : IntB.subranges()) {
815815 LaneBitmask BMask = SB.LaneMask;
816816 LaneBitmask Common = BMask & AMask;
817 if (Common == 0)
817 if (Common.none())
818818 continue;
819819
820820 DEBUG( dbgs() << "\t\tCopy_Merge " << PrintLaneMask(BMask)
821821 << " into " << PrintLaneMask(Common) << '\n');
822822 LaneBitmask BRest = BMask & ~AMask;
823823 LiveInterval::SubRange *CommonRange;
824 if (BRest != 0) {
824 if (!BRest.none()) {
825825 SB.LaneMask = BRest;
826826 DEBUG(dbgs() << "\t\tReduce Lane to " << PrintLaneMask(BRest)
827827 << '\n');
840840 addSegmentsWithValNo(*CommonRange, BSubValNo, SA, ASubValNo);
841841 AMask &= ~BMask;
842842 }
843 if (AMask != 0) {
843 if (!AMask.none()) {
844844 DEBUG(dbgs() << "\t\tNew Lane " << PrintLaneMask(AMask) << '\n');
845845 LiveRange *NewRange = IntB.createSubRange(Allocator, AMask);
846846 VNInfo *BSubValNo = NewRange->getNextValue(CopyIdx, Allocator);
10601060 SR.createDeadDef(DefIndex, Alloc);
10611061 MaxMask &= ~SR.LaneMask;
10621062 }
1063 if (MaxMask != 0) {
1063 if (!MaxMask.none()) {
10641064 LiveInterval::SubRange *SR = DstInt.createSubRange(Alloc, MaxMask);
10651065 SR->createDeadDef(DefIndex, Alloc);
10661066 }
11531153 if (SrcSubIdx != 0 && SrcLI.hasSubRanges()) {
11541154 LaneBitmask SrcMask = TRI->getSubRegIndexLaneMask(SrcSubIdx);
11551155 for (const LiveInterval::SubRange &SR : SrcLI.subranges()) {
1156 if ((SR.LaneMask & SrcMask) == 0)
1156 if ((SR.LaneMask & SrcMask).none())
11571157 continue;
11581158 if (SR.liveAt(Idx))
11591159 return false;
11741174 // The affected subregister segments can be removed.
11751175 LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(DstSubIdx);
11761176 for (LiveInterval::SubRange &SR : DstLI.subranges()) {
1177 if ((SR.LaneMask & DstMask) == 0)
1177 if ((SR.LaneMask & DstMask).none())
11781178 continue;
11791179
11801180 VNInfo *SVNI = SR.getVNInfoAt(RegIndex);
11931193 SlotIndex UseIdx = LIS->getInstructionIndex(MI);
11941194 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
11951195 bool isLive;
1196 if (UseMask != ~0u && DstLI.hasSubRanges()) {
1196 if (!UseMask.all() && DstLI.hasSubRanges()) {
11971197 isLive = false;
11981198 for (const LiveInterval::SubRange &SR : DstLI.subranges()) {
1199 if ((SR.LaneMask & UseMask) == 0)
1199 if ((SR.LaneMask & UseMask).none())
12001200 continue;
12011201 if (SR.liveAt(UseIdx)) {
12021202 isLive = true;
12311231 Mask = ~Mask;
12321232 bool IsUndef = true;
12331233 for (const LiveInterval::SubRange &S : Int.subranges()) {
1234 if ((S.LaneMask & Mask) == 0)
1234 if ((S.LaneMask & Mask).none())
12351235 continue;
12361236 if (S.liveAt(UseIdx)) {
12371237 IsUndef = false;
14571457 });
14581458 }
14591459
1460 ShrinkMask = 0;
1460 ShrinkMask = LaneBitmask::getNone();
14611461 ShrinkMainRange = false;
14621462
14631463 // Okay, attempt to join these two intervals. On failure, this returns false.
15151515 updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx());
15161516
15171517 // Shrink subregister ranges if necessary.
1518 if (ShrinkMask != 0) {
1518 if (!ShrinkMask.none()) {
15191519 LiveInterval &LI = LIS->getInterval(CP.getDstReg());
15201520 for (LiveInterval::SubRange &S : LI.subranges()) {
1521 if ((S.LaneMask & ShrinkMask) == 0)
1521 if ((S.LaneMask & ShrinkMask).none())
15221522 continue;
15231523 DEBUG(dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask)
15241524 << ")\n");
18161816 /// True once Pruned above has been computed.
18171817 bool PrunedComputed;
18181818
1819 Val() : Resolution(CR_Keep), WriteLanes(0), ValidLanes(0),
1819 Val() : Resolution(CR_Keep), WriteLanes(), ValidLanes(),
18201820 RedefVNI(nullptr), OtherVNI(nullptr), ErasableImplicitDef(false),
18211821 Pruned(false), PrunedComputed(false) {}
18221822
1823 bool isAnalyzed() const { return WriteLanes != 0; }
1823 bool isAnalyzed() const { return !WriteLanes.none(); }
18241824 };
18251825
18261826 /// One entry per value number in LI.
19371937
19381938 LaneBitmask JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef)
19391939 const {
1940 LaneBitmask L = 0;
1940 LaneBitmask L;
19411941 for (const MachineOperand &MO : DefMI->operands()) {
19421942 if (!MO.isReg() || MO.getReg() != Reg || !MO.isDef())
19431943 continue;
19751975 for (const LiveInterval::SubRange &S : LI.subranges()) {
19761976 // Transform lanemask to a mask in the joined live interval.
19771977 LaneBitmask SMask = TRI->composeSubRegIndexLaneMask(SubIdx, S.LaneMask);
1978 if ((SMask & LaneMask) == 0)
1978 if ((SMask & LaneMask).none())
19791979 continue;
19801980 LiveQueryResult LRQ = S.Query(Def);
19811981 ValueIn = LRQ.valueIn();
20152015 assert(!V.isAnalyzed() && "Value has already been analyzed!");
20162016 VNInfo *VNI = LR.getValNumInfo(ValNo);
20172017 if (VNI->isUnused()) {
2018 V.WriteLanes = ~0u;
2018 V.WriteLanes = LaneBitmask::getAll();
20192019 return CR_Keep;
20202020 }
20212021
20232023 const MachineInstr *DefMI = nullptr;
20242024 if (VNI->isPHIDef()) {
20252025 // Conservatively assume that all lanes in a PHI are valid.
2026 LaneBitmask Lanes = SubRangeJoin ? 1 : TRI->getSubRegIndexLaneMask(SubIdx);
2026 LaneBitmask Lanes = SubRangeJoin ? LaneBitmask(1)
2027 : TRI->getSubRegIndexLaneMask(SubIdx);
20272028 V.ValidLanes = V.WriteLanes = Lanes;
20282029 } else {
20292030 DefMI = Indexes->getInstructionFromIndex(VNI->def);
20302031 assert(DefMI != nullptr);
20312032 if (SubRangeJoin) {
20322033 // We don't care about the lanes when joining subregister ranges.
2033 V.WriteLanes = V.ValidLanes = 1;
2034 V.WriteLanes = V.ValidLanes = LaneBitmask(1);
20342035 if (DefMI->isImplicitDef()) {
2035 V.ValidLanes = 0;
2036 V.ValidLanes = LaneBitmask::getNone();
20362037 V.ErasableImplicitDef = true;
20372038 }
20382039 } else {
21052106 // predecessor, the PHI itself can't introduce any conflicts.
21062107 if (VNI->isPHIDef())
21072108 return CR_Merge;
2108 if (V.ValidLanes & OtherV.ValidLanes)
2109 if (!(V.ValidLanes & OtherV.ValidLanes).none())
21092110 // Overlapping lanes can't be resolved.
21102111 return CR_Impossible;
21112112 else
21502151 // We need the def for the subregister if there is nothing else live at the
21512152 // subrange at this point.
21522153 if (TrackSubRegLiveness
2153 && (V.WriteLanes & (OtherV.ValidLanes | OtherV.WriteLanes)) == 0)
2154 && (V.WriteLanes & (OtherV.ValidLanes | OtherV.WriteLanes)).none())
21542155 return CR_Replace;
21552156 return CR_Erase;
21562157 }
21902191 //
21912192 // Here OtherVNI will map to itself in [1;2), but to VNI in [2;5). CR_Replace
21922193 // handles this complex value mapping.
2193 if ((V.WriteLanes & OtherV.ValidLanes) == 0)
2194 if ((V.WriteLanes & OtherV.ValidLanes).none())
21942195 return CR_Replace;
21952196
21962197 // If the other live range is killed by DefMI and the live ranges are still
22112212 // possibility that no instructions actually read the clobbered lanes.
22122213 // If we're clobbering all the lanes in OtherVNI, at least one must be read.
22132214 // Otherwise Other.RI wouldn't be live here.
2214 if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes) == 0)
2215 if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes).none())
22152216 return CR_Impossible;
22162217
22172218 // We need to verify that no instructions are reading the clobbered lanes. To
22592260 Val &OtherV = Other.Vals[V.OtherVNI->id];
22602261 // We cannot erase an IMPLICIT_DEF if we don't have valid values for all
22612262 // its lanes.
2262 if ((OtherV.WriteLanes & ~V.ValidLanes) != 0 && TrackSubRegLiveness)
2263 if (!(OtherV.WriteLanes & ~V.ValidLanes).none() && TrackSubRegLiveness)
22632264 OtherV.ErasableImplicitDef = false;
22642265 OtherV.Pruned = true;
22652266 LLVM_FALLTHROUGH;
23202321 TaintedLanes &= ~OV.WriteLanes;
23212322 if (!OV.RedefVNI)
23222323 break;
2323 } while (TaintedLanes);
2324 } while (!TaintedLanes.none());
23242325 return true;
23252326 }
23262327
23332334 continue;
23342335 if (!MO.readsReg())
23352336 continue;
2336 if (Lanes & TRI->getSubRegIndexLaneMask(
2337 TRI->composeSubRegIndices(SubIdx, MO.getSubReg())))
2337 unsigned S = TRI->composeSubRegIndices(SubIdx, MO.getSubReg());
2338 if (!(Lanes & TRI->getSubRegIndexLaneMask(S)).none())
23382339 return true;
23392340 }
23402341 return false;
27212722 // LaneMask of subregisters common to subrange R and ToMerge.
27222723 LaneBitmask Common = RMask & LaneMask;
27232724 // There is nothing to do without common subregs.
2724 if (Common == 0)
2725 if (Common.none())
27252726 continue;
27262727
27272728 DEBUG(dbgs() << "\t\tCopy+Merge " << PrintLaneMask(RMask) << " into "
27302731 // they have to split into their own subrange.
27312732 LaneBitmask LRest = RMask & ~LaneMask;
27322733 LiveInterval::SubRange *CommonRange;
2733 if (LRest != 0) {
2734 if (!LRest.none()) {
27342735 R.LaneMask = LRest;
27352736 DEBUG(dbgs() << "\t\tReduce Lane to " << PrintLaneMask(LRest) << '\n');
27362737 // Duplicate SubRange for newly merged common stuff.
27452746 LaneMask &= ~RMask;
27462747 }
27472748
2748 if (LaneMask != 0) {
2749 if (!LaneMask.none()) {
27492750 DEBUG(dbgs() << "\t\tNew Lane " << PrintLaneMask(LaneMask) << '\n');
27502751 LI.createSubRangeFrom(Allocator, LaneMask, ToMerge);
27512752 }
27562757 LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
27572758 LiveInterval &LHS = LIS->getInterval(CP.getDstReg());
27582759 bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC());
2759 JoinVals RHSVals(RHS, CP.getSrcReg(), CP.getSrcIdx(), 0, NewVNInfo, CP, LIS,
2760 TRI, false, TrackSubRegLiveness);
2761 JoinVals LHSVals(LHS, CP.getDstReg(), CP.getDstIdx(), 0, NewVNInfo, CP, LIS,
2762 TRI, false, TrackSubRegLiveness);
2760 JoinVals RHSVals(RHS, CP.getSrcReg(), CP.getSrcIdx(), LaneBitmask::getNone(),
2761 NewVNInfo, CP, LIS, TRI, false, TrackSubRegLiveness);
2762 JoinVals LHSVals(LHS, CP.getDstReg(), CP.getDstIdx(), LaneBitmask::getNone(),
2763 NewVNInfo, CP, LIS, TRI, false, TrackSubRegLiveness);
27632764
27642765 DEBUG(dbgs() << "\t\tRHS = " << RHS
27652766 << "\n\t\tLHS = " << LHS
27852786 LaneBitmask Mask = DstIdx == 0 ? CP.getNewRC()->getLaneMask()
27862787 : TRI->getSubRegIndexLaneMask(DstIdx);
27872788 // LHS must support subregs or we wouldn't be in this codepath.
2788 assert(Mask != 0);
2789 assert(!Mask.none());
27892790 LHS.createSubRangeFrom(Allocator, Mask, LHS);
27902791 } else if (DstIdx != 0) {
27912792 // Transform LHS lanemasks to new register class if necessary.
31653166 // If subranges are still supported, then the same subregs
31663167 // should still be supported.
31673168 for (LiveInterval::SubRange &S : LI.subranges()) {
3168 assert((S.LaneMask & ~MaxMask) == 0);
3169 assert((S.LaneMask & ~MaxMask).none());
31693170 }
31703171 #endif
31713172 }
2525 static void increaseSetPressure(std::vector &CurrSetPressure,
2626 const MachineRegisterInfo &MRI, unsigned Reg,
2727 LaneBitmask PrevMask, LaneBitmask NewMask) {
28 assert((PrevMask & ~NewMask) == 0 && "Must not remove bits");
29 if (PrevMask != 0 || NewMask == 0)
28 assert((PrevMask & ~NewMask).none() && "Must not remove bits");
29 if (!PrevMask.none() || NewMask.none())
3030 return;
3131
3232 PSetIterator PSetI = MRI.getPressureSets(Reg);
3939 static void decreaseSetPressure(std::vector &CurrSetPressure,
4040 const MachineRegisterInfo &MRI, unsigned Reg,
4141 LaneBitmask PrevMask, LaneBitmask NewMask) {
42 assert((NewMask & !PrevMask) == 0 && "Must not add bits");
43 if (NewMask != 0 || PrevMask == 0)
42 //assert((NewMask & !PrevMask) == 0 && "Must not add bits");
43 if (!NewMask.none() || PrevMask.none())
4444 return;
4545
4646 PSetIterator PSetI = MRI.getPressureSets(Reg);
7272 dbgs() << "Live In: ";
7373 for (const RegisterMaskPair &P : LiveInRegs) {
7474 dbgs() << PrintVRegOrUnit(P.RegUnit, TRI);
75 if (P.LaneMask != ~0u)
75 if (!P.LaneMask.all())
7676 dbgs() << ':' << PrintLaneMask(P.LaneMask);
7777 dbgs() << ' ';
7878 }
8080 dbgs() << "Live Out: ";
8181 for (const RegisterMaskPair &P : LiveOutRegs) {
8282 dbgs() << PrintVRegOrUnit(P.RegUnit, TRI);
83 if (P.LaneMask != ~0u)
83 if (!P.LaneMask.all())
8484 dbgs() << ':' << PrintLaneMask(P.LaneMask);
8585 dbgs() << ' ';
8686 }
111111 void RegPressureTracker::increaseRegPressure(unsigned RegUnit,
112112 LaneBitmask PreviousMask,
113113 LaneBitmask NewMask) {
114 if (PreviousMask != 0 || NewMask == 0)
114 if (!PreviousMask.none() || NewMask.none())
115115 return;
116116
117117 PSetIterator PSetI = MRI->getPressureSets(RegUnit);
321321 unsigned RegUnit = Pair.RegUnit;
322322 if (TargetRegisterInfo::isVirtualRegister(RegUnit)
323323 && !RPTracker.hasUntiedDef(RegUnit))
324 increaseSetPressure(LiveThruPressure, *MRI, RegUnit, 0, Pair.LaneMask);
324 increaseSetPressure(LiveThruPressure, *MRI, RegUnit,
325 LaneBitmask::getNone(), Pair.LaneMask);
325326 }
326327 }
327328
331332 return Other.RegUnit == RegUnit;
332333 });
333334 if (I == RegUnits.end())
334 return 0;
335 return LaneBitmask::getNone();
335336 return I->LaneMask;
336337 }
337338
338339 static void addRegLanes(SmallVectorImpl &RegUnits,
339340 RegisterMaskPair Pair) {
340341 unsigned RegUnit = Pair.RegUnit;
341 assert(Pair.LaneMask != 0);
342 assert(!Pair.LaneMask.none());
342343 auto I = find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
343344 return Other.RegUnit == RegUnit;
344345 });
355356 return Other.RegUnit == RegUnit;
356357 });
357358 if (I == RegUnits.end()) {
358 RegUnits.push_back(RegisterMaskPair(RegUnit, 0));
359 RegUnits.push_back(RegisterMaskPair(RegUnit, LaneBitmask::getNone()));
359360 } else {
360 I->LaneMask = 0;
361 I->LaneMask = LaneBitmask::getNone();
361362 }
362363 }
363364
364365 static void removeRegLanes(SmallVectorImpl &RegUnits,
365366 RegisterMaskPair Pair) {
366367 unsigned RegUnit = Pair.RegUnit;
367 assert(Pair.LaneMask != 0);
368 assert(!Pair.LaneMask.none());
368369 auto I = find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
369370 return Other.RegUnit == RegUnit;
370371 });
371372 if (I != RegUnits.end()) {
372373 I->LaneMask &= ~Pair.LaneMask;
373 if (I->LaneMask == 0)
374 if (I->LaneMask.none())
374375 RegUnits.erase(I);
375376 }
376377 }
381382 bool(*Property)(const LiveRange &LR, SlotIndex Pos)) {
382383 if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
383384 const LiveInterval &LI = LIS.getInterval(RegUnit);
384 LaneBitmask Result = 0;
385 LaneBitmask Result;
385386 if (TrackLaneMasks && LI.hasSubRanges()) {
386387 for (const LiveInterval::SubRange &SR : LI.subranges()) {
387388 if (Property(SR, Pos))
388389 Result |= SR.LaneMask;
389390 }
390391 } else if (Property(LI, Pos)) {
391 Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit) : ~0u;
392 Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit)
393 : LaneBitmask::getAll();
392394 }
393395
394396 return Result;
398400 // for physical registers on targets with many registers (GPUs).
399401 if (LR == nullptr)
400402 return SafeDefault;
401 return Property(*LR, Pos) ? ~0u : 0;
403 return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
402404 }
403405 }
404406
406408 const MachineRegisterInfo &MRI,
407409 bool TrackLaneMasks, unsigned RegUnit,
408410 SlotIndex Pos) {
409 return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos, ~0u,
411 return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos,
412 LaneBitmask::getAll(),
410413 [](const LiveRange &LR, SlotIndex Pos) {
411414 return LR.liveAt(Pos);
412415 });
473476 void pushReg(unsigned Reg,
474477 SmallVectorImpl &RegUnits) const {
475478 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
476 addRegLanes(RegUnits, RegisterMaskPair(Reg, ~0u));
479 addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneBitmask::getAll()));
477480 } else if (MRI.isAllocatable(Reg)) {
478481 for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
479 addRegLanes(RegUnits, RegisterMaskPair(*Units, ~0u));
482 addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
480483 }
481484 }
482485
511514 addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneMask));
512515 } else if (MRI.isAllocatable(Reg)) {
513516 for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
514 addRegLanes(RegUnits, RegisterMaskPair(*Units, ~0u));
517 addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
515518 }
516519 }
517520
562565 // of a subregister def we need a read-undef flag.
563566 unsigned RegUnit = I->RegUnit;
564567 if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
565 AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask) == 0)
568 AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask).none())
566569 AddFlagsMI->setRegisterDefReadUndef(RegUnit);
567570
568571 LaneBitmask ActualDef = I->LaneMask & LiveAfter;
569 if (ActualDef == 0) {
572 if (ActualDef.none()) {
570573 I = Defs.erase(I);
571574 } else {
572575 I->LaneMask = ActualDef;
577580 LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
578581 Pos.getBaseIndex());
579582 LaneBitmask LaneMask = I->LaneMask & LiveBefore;
580 if (LaneMask == 0) {
583 if (LaneMask.none()) {
581584 I = Uses.erase(I);
582585 } else {
583586 I->LaneMask = LaneMask;
591594 continue;
592595 LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
593596 Pos.getDeadSlot());
594 if (LiveAfter == 0)
597 if (LiveAfter.none())
595598 AddFlagsMI->setRegisterDefReadUndef(RegUnit);
596599 }
597600 }
668671
669672 void RegPressureTracker::discoverLiveInOrOut(RegisterMaskPair Pair,
670673 SmallVectorImpl &LiveInOrOut) {
671 assert(Pair.LaneMask != 0);
674 assert(!Pair.LaneMask.none());
672675
673676 unsigned RegUnit = Pair.RegUnit;
674677 auto I = find_if(LiveInOrOut, [RegUnit](const RegisterMaskPair &Other) {
677680 LaneBitmask PrevMask;
678681 LaneBitmask NewMask;
679682 if (I == LiveInOrOut.end()) {
680 PrevMask = 0;
683 PrevMask = LaneBitmask::getNone();
681684 NewMask = Pair.LaneMask;
682685 LiveInOrOut.push_back(Pair);
683686 } else {
732735 LaneBitmask NewMask = PreviousMask & ~Def.LaneMask;
733736
734737 LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask;
735 if (LiveOut != 0) {
738 if (!LiveOut.none()) {
736739 discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
737740 // Retroactively model effects on pressure of the live out lanes.
738 increaseSetPressure(CurrSetPressure, *MRI, Reg, 0, LiveOut);
741 increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(),
742 LiveOut);
739743 PreviousMask = LiveOut;
740744 }
741745
742 if (NewMask == 0) {
746 if (NewMask.none()) {
743747 // Add a 0 entry to LiveUses as a marker that the complete vreg has become
744748 // dead.
745749 if (TrackLaneMasks && LiveUses != nullptr)
756760 // Generate liveness for uses.
757761 for (const RegisterMaskPair &Use : RegOpers.Uses) {
758762 unsigned Reg = Use.RegUnit;
759 assert(Use.LaneMask != 0);
763 assert(!Use.LaneMask.none());
760764 LaneBitmask PreviousMask = LiveRegs.insert(Use);
761765 LaneBitmask NewMask = PreviousMask | Use.LaneMask;
762766 if (NewMask == PreviousMask)
763767 continue;
764768
765769 // Did the register just become live?
766 if (PreviousMask == 0) {
770 if (PreviousMask.none()) {
767771 if (LiveUses != nullptr) {
768772 if (!TrackLaneMasks) {
769773 addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
774778 bool IsRedef = I != LiveUses->end();
775779 if (IsRedef) {
776780 // ignore re-defs here...
777 assert(I->LaneMask == 0);
781 assert(I->LaneMask.none());
778782 removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
779783 } else {
780784 addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
785789 // Discover live outs if this may be the first occurance of this register.
786790 if (RequireIntervals) {
787791 LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx);
788 if (LiveOut != 0)
792 if (!LiveOut.none())
789793 discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
790794 }
791795 }
796800 for (const RegisterMaskPair &Def : RegOpers.Defs) {
797801 unsigned RegUnit = Def.RegUnit;
798802 if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
799 (LiveRegs.contains(RegUnit) & Def.LaneMask) == 0)
803 (LiveRegs.contains(RegUnit) & Def.LaneMask).none())
800804 UntiedDefs.insert(RegUnit);
801805 }
802806 }
864868 unsigned Reg = Use.RegUnit;
865869 LaneBitmask LiveMask = LiveRegs.contains(Reg);
866870 LaneBitmask LiveIn = Use.LaneMask & ~LiveMask;
867 if (LiveIn != 0) {
871 if (!LiveIn.none()) {
868872 discoverLiveIn(RegisterMaskPair(Reg, LiveIn));
869873 increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn);
870874 LiveRegs.insert(RegisterMaskPair(Reg, LiveIn));
872876 // Kill liveness at last uses.
873877 if (RequireIntervals) {
874878 LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
875 if (LastUseMask != 0) {
879 if (!LastUseMask.none()) {
876880 LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask));
877881 decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask);
878882 }
11851189 unsigned SubRegIdx = MO.getSubReg();
11861190 LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
11871191 LastUseMask &= ~UseMask;
1188 if (LastUseMask == 0)
1189 return 0;
1192 if (LastUseMask.none())
1193 return LaneBitmask::getNone();
11901194 }
11911195 }
11921196 return LastUseMask;
11951199 LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit,
11961200 SlotIndex Pos) const {
11971201 assert(RequireIntervals);
1198 return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, ~0u,
1202 return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
1203 LaneBitmask::getAll(),
11991204 [](const LiveRange &LR, SlotIndex Pos) {
12001205 return LR.liveAt(Pos);
12011206 });
12051210 SlotIndex Pos) const {
12061211 assert(RequireIntervals);
12071212 return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit,
1208 Pos.getBaseIndex(), 0,
1213 Pos.getBaseIndex(), LaneBitmask::getNone(),
12091214 [](const LiveRange &LR, SlotIndex Pos) {
12101215 const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
12111216 return S != nullptr && S->end == Pos.getRegSlot();
12151220 LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit,
12161221 SlotIndex Pos) const {
12171222 assert(RequireIntervals);
1218 return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos, 0u,
1223 return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
1224 LaneBitmask::getNone(),
12191225 [](const LiveRange &LR, SlotIndex Pos) {
12201226 const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
12211227 return S != nullptr && S->start < Pos.getRegSlot(true) &&
12461252 for (const RegisterMaskPair &Use : RegOpers.Uses) {
12471253 unsigned Reg = Use.RegUnit;
12481254 LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
1249 if (LastUseMask == 0)
1255 if (LastUseMask.none())
12501256 continue;
12511257 // The LastUseMask is queried from the liveness information of instruction
12521258 // which may be further down the schedule. Some lanes may actually not be
12561262 SlotIndex CurrIdx = getCurrSlot();
12571263 LastUseMask
12581264 = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
1259 if (LastUseMask == 0)
1265 if (LastUseMask.none())
12601266 continue;
12611267
12621268 LaneBitmask LiveMask = LiveRegs.contains(Reg);
3333 void RegScavenger::setRegUsed(unsigned Reg, LaneBitmask LaneMask) {
3434 for (MCRegUnitMaskIterator RUI(Reg, TRI); RUI.isValid(); ++RUI) {
3535 LaneBitmask UnitMask = (*RUI).second;
36 if (UnitMask == 0 || (LaneMask & UnitMask) != 0)
36 if (UnitMask.none() || !(LaneMask & UnitMask).none())
3737 RegUnitsAvailable.reset((*RUI).first);
3838 }
3939 }
183183 unsigned MergedID = ~0u;
184184 for (RenameIndependentSubregs::SubRangeInfo &SRInfo : SubRangeInfos) {
185185 const LiveInterval::SubRange &SR = *SRInfo.SR;
186 if ((SR.LaneMask & LaneMask) == 0)
186 if ((SR.LaneMask & LaneMask).none())
187187 continue;
188188 SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent());
189189 Pos = MO.isDef() ? Pos.getRegSlot(MO.isEarlyClobber())
227227 unsigned ID = ~0u;
228228 for (const SubRangeInfo &SRInfo : SubRangeInfos) {
229229 const LiveInterval::SubRange &SR = *SRInfo.SR;
230 if ((SR.LaneMask & LaneMask) == 0)
230 if ((SR.LaneMask & LaneMask).none())
231231 continue;
232232 const VNInfo *VNI = SR.getVNInfoAt(Pos);
233233 if (VNI == nullptr)
397397 // No point in tracking lanemasks if we don't have interesting subregisters.
398398 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
399399 if (!RC.HasDisjunctSubRegs)
400 return ~0u;
400 return LaneBitmask::getAll();
401401
402402 unsigned SubReg = MO.getSubReg();
403403 if (SubReg == 0)
423423 DefLaneMask = getLaneMaskForMO(MO);
424424 // If we have a flag, none of the lane values comes from an
425425 // earlier instruction.
426 KillLaneMask = IsKill ? ~0u : DefLaneMask;
426 KillLaneMask = IsKill ? LaneBitmask::getAll() : DefLaneMask;
427427
428428 // Clear undef flag, we'll re-add it later once we know which subregister
429429 // Def is first.
430430 MO.setIsUndef(false);
431431 } else {
432 DefLaneMask = ~0u;
433 KillLaneMask = ~0u;
432 DefLaneMask = LaneBitmask::getAll();
433 KillLaneMask = LaneBitmask::getAll();
434434 }
435435
436436 if (MO.isDead()) {
443443 E = CurrentVRegUses.end(); I != E; /*empty*/) {
444444 LaneBitmask LaneMask = I->LaneMask;
445445 // Ignore uses of other lanes.
446 if ((LaneMask & KillLaneMask) == 0) {
446 if ((LaneMask & KillLaneMask).none()) {
447447 ++I;
448448 continue;
449449 }
450450
451 if ((LaneMask & DefLaneMask) != 0) {
451 if (!(LaneMask & DefLaneMask).none()) {
452452 SUnit *UseSU = I->SU;
453453 MachineInstr *Use = UseSU->getInstr();
454454 SDep Dep(SU, SDep::Data, Reg);
460460
461461 LaneMask &= ~KillLaneMask;
462462 // If we found a Def for all lanes of this use, remove it from the list.
463 if (LaneMask != 0) {
463 if (!LaneMask.none()) {
464464 I->LaneMask = LaneMask;
465465 ++I;
466466 } else
483483 for (VReg2SUnit &V2SU : make_range(CurrentVRegDefs.find(Reg),
484484 CurrentVRegDefs.end())) {
485485 // Ignore defs for other lanes.
486 if ((V2SU.LaneMask & LaneMask) == 0)
486 if ((V2SU.LaneMask & LaneMask).none())
487487 continue;
488488 // Add an output dependence.
489489 SUnit *DefSU = V2SU.SU;
506506 LaneBitmask NonOverlapMask = V2SU.LaneMask & ~LaneMask;
507507 V2SU.SU = SU;
508508 V2SU.LaneMask = OverlapMask;
509 if (NonOverlapMask != 0)
509 if (!NonOverlapMask.none())
510510 CurrentVRegDefs.insert(VReg2SUnit(Reg, NonOverlapMask, DefSU));
511511 }
512512 // If there was no CurrentVRegDefs entry for some lanes yet, create one.
513 if (LaneMask != 0)
513 if (!LaneMask.none())
514514 CurrentVRegDefs.insert(VReg2SUnit(Reg, LaneMask, SU));
515515 }
516516
526526 unsigned Reg = MO.getReg();
527527
528528 // Remember the use. Data dependencies will be added when we find the def.
529 LaneBitmask LaneMask = TrackLaneMasks ? getLaneMaskForMO(MO) : ~0u;
529 LaneBitmask LaneMask = TrackLaneMasks ? getLaneMaskForMO(MO)
530 : LaneBitmask::getAll();
530531 CurrentVRegUses.insert(VReg2SUnitOperIdx(Reg, LaneMask, OperIdx, SU));
531532
532533 // Add antidependences to the following defs of the vreg.
534535 CurrentVRegDefs.end())) {
535536 // Ignore defs for unrelated lanes.
536537 LaneBitmask PrevDefLaneMask = V2SU.LaneMask;
537 if ((PrevDefLaneMask & LaneMask) == 0)
538 if ((PrevDefLaneMask & LaneMask).none())
538539 continue;
539540 if (V2SU.SU == SU)
540541 continue;
411411 // register, we need to check which subranges need to be updated.
412412 const MachineInstr *DefMI = LIS.getInstructionFromIndex(Def);
413413 assert(DefMI != nullptr);
414 LaneBitmask LM = 0;
414 LaneBitmask LM;
415415 for (const MachineOperand &DefOp : DefMI->defs()) {
416416 unsigned R = DefOp.getReg();
417417 if (R != LI.reg)
424424 }
425425 }
426426 for (LiveInterval::SubRange &S : LI.subranges())
427 if (S.LaneMask & LM)
427 if (!(S.LaneMask & LM).none())
428428 S.createDeadDef(Def, LIS.getVNInfoAllocator());
429429 }
430430 }
11011101 LiveInterval &PLI = Edit->getParent();
11021102 // Need the cast because the inputs to ?: would otherwise be deemed
11031103 // "incompatible": SubRange vs LiveInterval.
1104 LiveRange &PSR = (LM != ~0u) ? getSubRangeForMask(LM, PLI)
1105 : static_cast(PLI);
1104 LiveRange &PSR = !LM.all() ? getSubRangeForMask(LM, PLI)
1105 : static_cast(PLI);
11061106 if (PSR.liveAt(LastUse))
11071107 LRC.extend(LR, End, /*PhysReg=*/0, Undefs);
11081108 }
11251125 LiveRangeCalc &LRC = getLRCalc(RegIdx);
11261126 MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
11271127 if (!removeDeadSegment(V->def, LI))
1128 extendPHIRange(B, LRC, LI, ~0u, /*Undefs=*/{});
1128 extendPHIRange(B, LRC, LI, LaneBitmask::getAll(), /*Undefs=*/{});
11291129 }
11301130
11311131 SmallVector Undefs;
12281228 LaneBitmask LM = Sub != 0 ? TRI.getSubRegIndexLaneMask(Sub)
12291229 : MRI.getMaxLaneMaskForVReg(Reg);
12301230 for (LiveInterval::SubRange &S : LI.subranges()) {
1231 if (!(S.LaneMask & LM))
1231 if ((S.LaneMask & LM).none())
12321232 continue;
12331233 // The problem here can be that the new register may have been created
12341234 // for a partially defined original register. For example:
2929 TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID,
3030 regclass_iterator RCB, regclass_iterator RCE,
3131 const char *const *SRINames,
32 const unsigned *SRILaneMasks,
33 unsigned SRICoveringLanes)
32 const LaneBitmask *SRILaneMasks,
33 LaneBitmask SRICoveringLanes)
3434 : InfoDesc(ID), SubRegIndexNames(SRINames),
3535 SubRegIndexLaneMasks(SRILaneMasks),
3636 RegClassBegin(RCB), RegClassEnd(RCE),
123123 } else {
124124 OS << PrintRegUnit(Unit, TRI);
125125 }
126 });
127 }
128
129 Printable PrintLaneMask(LaneBitmask LaneMask) {
130 return Printable([LaneMask](raw_ostream &OS) {
131 OS << format("%08X", LaneMask);
132126 });
133127 }
134128
265265 SlotIndex MBBBegin = MBBI->first;
266266 // Advance all subrange iterators so that their end position is just
267267 // behind MBBBegin (or the iterator is at the end).
268 LaneBitmask LaneMask = 0;
268 LaneBitmask LaneMask;
269269 for (auto &RangeIterPair : SubRanges) {
270270 const LiveInterval::SubRange *SR = RangeIterPair.first;
271271 LiveInterval::const_iterator &SRI = RangeIterPair.second;
276276 if (SRI->start <= MBBBegin)
277277 LaneMask |= SR->LaneMask;
278278 }
279 if (LaneMask == 0)
279 if (LaneMask.none())
280280 continue;
281281 MachineBasicBlock *MBB = MBBI->second;
282282 MBB->addLiveIn(PhysReg, LaneMask);
341341 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
342342 // See if any of the relevant subregister liveranges is defined at this point.
343343 for (const LiveInterval::SubRange &SR : LI.subranges()) {
344 if ((SR.LaneMask & UseMask) != 0 && SR.liveAt(BaseIndex))
344 if (!(SR.LaneMask & UseMask).none() && SR.liveAt(BaseIndex))
345345 return false;
346346 }
347347 return true;
1111 //===----------------------------------------------------------------------===//
1212
1313 #include "llvm/MC/MCRegisterInfo.h"
14 #include "llvm/Support/Format.h"
15 #include "llvm/Support/raw_ostream.h"
1416
1517 using namespace llvm;
1618
10261026 return RC;
10271027
10281028 // We can assume that each lane corresponds to one 32-bit register.
1029 unsigned Count = countPopulation(getSubRegIndexLaneMask(SubIdx));
1029 LaneBitmask::Type Mask = getSubRegIndexLaneMask(SubIdx).getAsInteger();
1030 unsigned Count = countPopulation(Mask);
10301031 if (isSGPRClass(RC)) {
10311032 switch (Count) {
10321033 case 1:
233233 RegisterSet LiveIns;
234234 RegisterSet Tmp;
235235 for (auto I : B.liveins()) {
236 if (I.LaneMask == ~LaneBitmask(0)) {
236 if (I.LaneMask.all()) {
237237 Tmp.insert({I.PhysReg,0});
238238 continue;
239239 }
240240 for (MCSubRegIndexIterator S(I.PhysReg, &TRI); S.isValid(); ++S) {
241241 LaneBitmask M = TRI.getSubRegIndexLaneMask(S.getSubRegIndex());
242 if (M & I.LaneMask)
242 if (!(M & I.LaneMask).none())
243243 Tmp.insert({S.getSubReg(), 0});
244244 }
245245 }
368368 if (!TargetRegisterInfo::isVirtualRegister(DR) || DR != Reg)
369369 return false;
370370 LaneBitmask SLM = getLaneMask(DR, DSR);
371 return (SLM & LM) != 0;
371 return !(SLM & LM).none();
372372 };
373373
374374 // The splitting step will create pairs of predicated definitions without
2929 namespace rdf {
3030
3131 raw_ostream &operator<< (raw_ostream &OS, const PrintLaneMaskOpt &P) {
32 if (P.Mask != ~LaneBitmask(0))
32 if (!P.Mask.all())
3333 OS << ':' << PrintLaneMask(P.Mask);
3434 return OS;
3535 }
661661 RegisterRef NR = normalize(RR);
662662 auto F = Masks.find(NR.Reg);
663663 if (F != Masks.end()) {
664 if (F->second & NR.Mask)
664 if (!(F->second & NR.Mask).none())
665665 return true;
666666 }
667667 if (CheckUnits) {
675675 bool RegisterAggr::hasCoverOf(RegisterRef RR) const {
676676 // Always have a cover for empty lane mask.
677677 RegisterRef NR = normalize(RR);
678 if (!NR.Mask)
678 if (NR.Mask.none())
679679 return true;
680680 auto F = Masks.find(NR.Reg);
681681 if (F == Masks.end())
716716 if (F == Masks.end())
717717 return *this;
718718 LaneBitmask NewM = F->second & ~NR.Mask;
719 if (NewM == LaneBitmask(0))
719 if (NewM.none())
720720 Masks.erase(F);
721721 else
722722 F->second = NewM;
10881088 RegisterRef DataFlowGraph::restrictRef(RegisterRef AR, RegisterRef BR) const {
10891089 if (AR.Reg == BR.Reg) {
10901090 LaneBitmask M = AR.Mask & BR.Mask;
1091 return M ? RegisterRef(AR.Reg, M) : RegisterRef();
1091 return !M.none() ? RegisterRef(AR.Reg, M) : RegisterRef();
10921092 }
10931093 #ifndef NDEBUG
10941094 RegisterRef NAR = normalizeRef(AR);
12101210 // This can happen when the register has only one unit, or when the
12111211 // unit corresponds to explicit aliasing. In such cases, the lane mask
12121212 // from RegisterRef should be ignored.
1213 if (!PA.second || !PB.second)
1213 if (PA.second.none() || PB.second.none())
12141214 return true;
12151215
12161216 // At this point the common unit corresponds to a subregister. The lane
12201220 // while the lane mask of r2 in d1 may be 0b0001.
12211221 LaneBitmask LA = PA.second & RA.Mask;
12221222 LaneBitmask LB = PB.second & RB.Mask;
1223 if (LA != 0 && LB != 0) {
1223 if (!LA.none() && !LB.none()) {
12241224 unsigned Root = *MCRegUnitRootIterator(PA.first, &TRI);
12251225 // If register units were guaranteed to only have 1 bit in any lane
12261226 // mask, the code below would not be necessary. This is because LA
12311231 const TargetRegisterClass &RC = *TRI.getMinimalPhysRegClass(Root);
12321232 LaneBitmask MaskA = TRI.reverseComposeSubRegIndexLaneMask(SubA, LA);
12331233 LaneBitmask MaskB = TRI.reverseComposeSubRegIndexLaneMask(SubB, LB);
1234 if (MaskA & MaskB & RC.LaneMask)
1234 if (!(MaskA & MaskB & RC.LaneMask).none())
12351235 return true;
12361236 }
12371237
402402 LaneBitmask Mask;
403403
404404 RegisterRef() : RegisterRef(0) {}
405 explicit RegisterRef(RegisterId R, LaneBitmask M = ~LaneBitmask(0))
406 : Reg(R), Mask(R != 0 ? M : 0) {}
407 operator bool() const { return Reg != 0 && Mask != LaneBitmask(0); }
405 explicit RegisterRef(RegisterId R, LaneBitmask M = LaneBitmask::getAll())
406 : Reg(R), Mask(R != 0 ? M : LaneBitmask::getNone()) {}
407 operator bool() const { return Reg != 0 && !Mask.none(); }
408408 bool operator== (const RegisterRef &RR) const {
409409 return Reg == RR.Reg && Mask == RR.Mask;
410410 }
457457 uint32_t find(T Val) const {
458458 auto F = llvm::find(Map, Val);
459459 assert(F != Map.end());
460 return *F;
460 return F - Map.begin();
461461 }
462462 private:
463463 std::vector Map;
467467 LaneMaskIndex() = default;
468468
469469 LaneBitmask getLaneMaskForIndex(uint32_t K) const {
470 return K == 0 ? ~LaneBitmask(0) : get(K);
470 return K == 0 ? LaneBitmask::getAll() : get(K);
471471 }
472472 uint32_t getIndexForLaneMask(LaneBitmask LM) {
473 assert(LM != LaneBitmask(0));
474 return LM == ~LaneBitmask(0) ? 0 : insert(LM);
473 assert(!LM.none());
474 return LM.all() ? 0 : insert(LM);
475475 }
476476 uint32_t getIndexForLaneMask(LaneBitmask LM) const {
477 assert(LM != LaneBitmask(0));
478 return LM == ~LaneBitmask(0) ? 0 : find(LM);
477 assert(!LM.none());
478 return LM.all() ? 0 : find(LM);
479479 }
480480 PackedRegisterRef pack(RegisterRef RR) {
481481 return { RR.Reg, getIndexForLaneMask(RR.Mask) };
658658 RegisterRef UR = DFG.normalizeRef(getRestrictedRegRef(PUA));
659659 for (const std::pair &T : RUs) {
660660 // Check if T.first aliases UR?
661 LaneBitmask M = 0;
661 LaneBitmask M;
662662 for (std::pair P : T.second)
663663 M |= P.second;
664664
709709 }
710710 do {
711711 LaneBitmask M = TRI.getSubRegIndexLaneMask(S.getSubRegIndex());
712 if (M & P.second)
712 if (!(M & P.second).none())
713713 LV.push_back(RegisterRef(S.getSubReg()));
714714 ++S;
715715 } while (S.isValid());
758758 }
759759 do {
760760 LaneBitmask M = TRI.getSubRegIndexLaneMask(S.getSubRegIndex());
761 if (M & I.LaneMask)
761 if (!(M & I.LaneMask).none())
762762 LV.set(S.getSubReg());
763763 ++S;
764764 } while (S.isValid());
10001000 RegisterAggr &Local = LiveMap[B];
10011001 RefMap &LON = PhiLON[B];
10021002 for (auto &R : LON) {
1003 LaneBitmask M = 0;
1003 LaneBitmask M;
10041004 for (auto P : R.second)
10051005 M |= P.second;
10061006 Local.insert(RegisterRef(R.first,M));
5050 //===----------------------------------------------------------------------===//
5151
5252 CodeGenSubRegIndex::CodeGenSubRegIndex(Record *R, unsigned Enum)
53 : TheDef(R), EnumValue(Enum), LaneMask(0), AllSuperRegsCovered(true) {
53 : TheDef(R), EnumValue(Enum), AllSuperRegsCovered(true) {
5454 Name = R->getName();
5555 if (R->getValue("Namespace"))
5656 Namespace = R->getValueAsString("Namespace");
6161 CodeGenSubRegIndex::CodeGenSubRegIndex(StringRef N, StringRef Nspace,
6262 unsigned Enum)
6363 : TheDef(nullptr), Name(N), Namespace(Nspace), Size(-1), Offset(-1),
64 EnumValue(Enum), LaneMask(0), AllSuperRegsCovered(true) {
64 EnumValue(Enum), AllSuperRegsCovered(true) {
6565 }
6666
6767 std::string CodeGenSubRegIndex::getQualifiedName() const {
101101 }
102102 }
103103
104 unsigned CodeGenSubRegIndex::computeLaneMask() const {
104 LaneBitmask CodeGenSubRegIndex::computeLaneMask() const {
105105 // Already computed?
106 if (LaneMask)
106 if (!LaneMask.none())
107107 return LaneMask;
108108
109109 // Recursion guard, shouldn't be required.
110 LaneMask = ~0u;
110 LaneMask = LaneBitmask::getAll();
111111
112112 // The lane mask is simply the union of all sub-indices.
113 unsigned M = 0;
113 LaneBitmask M;
114114 for (const auto &C : Composed)
115115 M |= C.second->computeLaneMask();
116 assert(M && "Missing lane mask, sub-register cycle?");
116 assert(!M.none() && "Missing lane mask, sub-register cycle?");
117117 LaneMask = M;
118118 return LaneMask;
119119 }
677677 : TheDef(R),
678678 Name(R->getName()),
679679 TopoSigs(RegBank.getNumTopoSigs()),
680 EnumValue(-1),
681 LaneMask(0) {
680 EnumValue(-1) {
682681 // Rename anonymous register classes.
683682 if (R->getName().size() > 9 && R->getName()[9] == '.') {
684683 static unsigned AnonCounter = 0;
11921191 // First assign individual bits to all the leaf indices.
11931192 unsigned Bit = 0;
11941193 // Determine mask of lanes that cover their registers.
1195 CoveringLanes = ~0u;
1194 CoveringLanes = LaneBitmask::getAll();
11961195 for (auto &Idx : SubRegIndices) {
11971196 if (Idx.getComposites().empty()) {
11981197 if (Bit > 32) {
12001199 Twine("Ran out of lanemask bits to represent subregister ")
12011200 + Idx.getName());
12021201 }
1203 Idx.LaneMask = 1u << Bit;
1202 Idx.LaneMask = LaneBitmask(1 << Bit);
12041203 ++Bit;
12051204 } else {
1206 Idx.LaneMask = 0;
1205 Idx.LaneMask = LaneBitmask::getNone();
12071206 }
12081207 }
12091208
12221221 // Moving from a class with no subregisters we just had a single lane:
12231222 // The subregister must be a leaf subregister and only occupies 1 bit.
12241223 // Move the bit from the class without subregisters into that position.
1225 unsigned DstBit = Log2_32(Idx.LaneMask);
1226 assert(Idx.LaneMask == 1u << DstBit && "Must be a leaf subregister");
1227 MaskRolPair MaskRol = { 1, (uint8_t)DstBit };
1224 static_assert(sizeof(Idx.LaneMask.getAsInteger()) == 4,
1225 "Change Log2_32 to a proper one");
1226 unsigned DstBit = Log2_32(Idx.LaneMask.getAsInteger());
1227 assert(Idx.LaneMask == LaneBitmask(1 << DstBit) &&
1228 "Must be a leaf subregister");
1229 MaskRolPair MaskRol = { LaneBitmask(1), (uint8_t)DstBit };
12281230 LaneTransforms.push_back(MaskRol);
12291231 } else {
12301232 // Go through all leaf subregisters and find the ones that compose with
12381240 continue;
12391241 // Replicate the behaviour from the lane mask generation loop above.
12401242 unsigned SrcBit = NextBit;
1241 unsigned SrcMask = 1u << SrcBit;
1242 if (NextBit < 31)
1243 LaneBitmask SrcMask = LaneBitmask(1 << SrcBit);
1244 if (NextBit < LaneBitmask::BitWidth-1)
12431245 ++NextBit;
12441246 assert(Idx2.LaneMask == SrcMask);
12451247
12521254 assert(Composite->getComposites().empty());
12531255
12541256 // Create Mask+Rotate operation and merge with existing ops if possible.
1255 unsigned DstBit = Log2_32(Composite->LaneMask);
1257 static_assert(sizeof(Composite->LaneMask.getAsInteger()) == 4,
1258 "Change Log2_32 to a proper one");
1259 unsigned DstBit = Log2_32(Composite->LaneMask.getAsInteger());
12561260 int Shift = DstBit - SrcBit;
1257 uint8_t RotateLeft = Shift >= 0 ? (uint8_t)Shift : 32+Shift;
1261 uint8_t RotateLeft = Shift >= 0 ? (uint8_t)Shift
1262 : LaneBitmask::BitWidth + Shift;
12581263 for (auto &I : LaneTransforms) {
12591264 if (I.RotateLeft == RotateLeft) {
12601265 I.Mask |= SrcMask;
1261 SrcMask = 0;
1266 SrcMask = LaneBitmask::getNone();
12621267 }
12631268 }
1264 if (SrcMask != 0) {
1269 if (!SrcMask.none()) {
12651270 MaskRolPair MaskRol = { SrcMask, RotateLeft };
12661271 LaneTransforms.push_back(MaskRol);
12671272 }
12721277 // 0xffffffff (including some irrelevant invalid bits) so that it should
12731278 // merge with more entries later while compressing the table.
12741279 if (LaneTransforms.size() == 1)
1275 LaneTransforms[0].Mask = ~0u;
1280 LaneTransforms[0].Mask = LaneBitmask::getAll();
12761281
12771282 // Further compression optimization: For invalid compositions resulting
12781283 // in a sequence with 0 entries we can just pick any other. Choose
12791284 // Mask 0xffffffff with Rotation 0.
12801285 if (LaneTransforms.size() == 0) {
1281 MaskRolPair P = { ~0u, 0 };
1286 MaskRolPair P = { LaneBitmask::getAll(), 0 };
12821287 LaneTransforms.push_back(P);
12831288 }
12841289 }
12881293
12891294 // Inherit lanes from composites.
12901295 for (const auto &Idx : SubRegIndices) {
1291 unsigned Mask = Idx.computeLaneMask();
1296 LaneBitmask Mask = Idx.computeLaneMask();
12921297 // If some super-registers without CoveredBySubRegs use this index, we can
12931298 // no longer assume that the lanes are covering their registers.
12941299 if (!Idx.AllSuperRegsCovered)
12971302
12981303 // Compute lane mask combinations for register classes.
12991304 for (auto &RegClass : RegClasses) {
1300 unsigned LaneMask = 0;
1305 LaneBitmask LaneMask;
13011306 for (const auto &SubRegIndex : SubRegIndices) {
13021307 if (RegClass.getSubClassWithSubReg(&SubRegIndex) == nullptr)
13031308 continue;
13061311
13071312 // For classes without any subregisters set LaneMask to 1 instead of 0.
13081313 // This makes it easier for client code to handle classes uniformly.
1309 if (LaneMask == 0)
1310 LaneMask = 1;
1314 if (LaneMask.none())
1315 LaneMask = LaneBitmask(1);
13111316
13121317 RegClass.LaneMask = LaneMask;
13131318 }
18061811 for (auto &Register : Registers) {
18071812 // Create an initial lane mask for all register units.
18081813 const auto &RegUnits = Register.getRegUnits();
1809 CodeGenRegister::RegUnitLaneMaskList RegUnitLaneMasks(RegUnits.count(), 0);
1814 CodeGenRegister::RegUnitLaneMaskList
1815 RegUnitLaneMasks(RegUnits.count(), LaneBitmask::getNone());
18101816 // Iterate through SubRegisters.
18111817 typedef CodeGenRegister::SubRegMap SubRegMap;
18121818 const SubRegMap &SubRegs = Register.getSubRegs();
18191825 continue;
18201826 CodeGenSubRegIndex *SubRegIndex = S->first;
18211827 const CodeGenRegister *SubRegister = S->second;
1822 unsigned LaneMask = SubRegIndex->LaneMask;
1828 LaneBitmask LaneMask = SubRegIndex->LaneMask;
18231829 // Distribute LaneMask to Register Units touched.
18241830 for (unsigned SUI : SubRegister->getRegUnits()) {
18251831 bool Found = false;
2525 #include "llvm/ADT/StringMap.h"
2626 #include "llvm/ADT/StringRef.h"
2727 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/MC/LaneBitmask.h"
2829 #include "llvm/Support/ErrorHandling.h"
2930 #include "llvm/TableGen/Record.h"
3031 #include "llvm/TableGen/SetTheory.h"
4647 /// Mask the bits specified in Mask, then rotate them Rol bits to the left
4748 /// assuming a wraparound at 32bits.
4849 struct MaskRolPair {
49 unsigned Mask;
50 LaneBitmask Mask;
5051 uint8_t RotateLeft;
5152
5253 bool operator==(const MaskRolPair Other) const {
6768 uint16_t Size;
6869 uint16_t Offset;
6970 const unsigned EnumValue;
70 mutable unsigned LaneMask;
71 mutable LaneBitmask LaneMask;
7172 mutable SmallVector CompositionLaneMaskTransform;
7273
7374 // Are all super-registers containing this SubRegIndex covered by their
119120 const CompMap &getComposites() const { return Composed; }
120121
121122 // Compute LaneMask from Composed. Return LaneMask.
122 unsigned computeLaneMask() const;
123 LaneBitmask computeLaneMask() const;
123124
124125 private:
125126 CompMap Composed;
205206
206207 // List of register units in ascending order.
207208 typedef SparseBitVector<> RegUnitList;
208 typedef SmallVector<unsigned, 16> RegUnitLaneMaskList;
209 typedef SmallVector<LaneBitmask, 16> RegUnitLaneMaskList;
209210
210211 // How many entries in RegUnitList are native?
211212 RegUnitList NativeRegUnits;
214215 // This is only valid after computeSubRegs() completes.
215216 const RegUnitList &getRegUnits() const { return RegUnits; }
216217
217 ArrayRef<unsigned> getRegUnitLaneMasks() const {
218 ArrayRef<LaneBitmask> getRegUnitLaneMasks() const {
218219 return makeArrayRef(RegUnitLaneMasks).slice(0, NativeRegUnits.count());
219220 }
220221
315316 std::string AltOrderSelect;
316317 uint8_t AllocationPriority;
317318 /// Contains the combination of the lane masks of all subregisters.
318 unsigned LaneMask;
319 LaneBitmask LaneMask;
319320 /// True if there are at least 2 subregisters which do not interfere.
320321 bool HasDisjunctSubRegs;
321322 bool CoveredBySubRegs;
732733 // Bit mask of lanes that cover their registers. A sub-register index whose
733734 // LaneMask is contained in CoveringLanes will be completely covered by
734735 // another sub-register with the same or larger lane mask.
735 unsigned CoveringLanes;
736 LaneBitmask CoveringLanes;
736737 };
737738
738739 } // end namespace llvm
577577 // 0 differential which means we can't encode repeated elements.
578578
579579 typedef SmallVector DiffVec;
580 typedef SmallVector<unsigned, 4> MaskVec;
580 typedef SmallVector<LaneBitmask, 4> MaskVec;
581581
582582 // Differentially encode a sequence of numbers into V. The starting value and
583583 // terminating 0 are not added to V, so it will have the same size as List.
610610 OS << Val;
611611 }
612612
613 static void printMask(raw_ostream &OS, unsigned Val) {
614 OS << format("0x%08X", Val);
613 static void printMask(raw_ostream &OS, LaneBitmask Val) {
614 OS << "LaneBitmask(0x" << PrintLaneMask(Val) << ')';
615615 }
616616
617617 // Try to combine Idx's compose map into Vec if it is compatible.
738738 }
739739
740740 OS << " struct MaskRolOp {\n"
741 " unsigned Mask;\n"
741 " LaneBitmask Mask;\n"
742742 " uint8_t RotateLeft;\n"
743743 " };\n"
744744 " static const MaskRolOp LaneMaskComposeSequences[] = {\n";
748748 const SmallVectorImpl &Sequence = Sequences[s];
749749 for (size_t p = 0, pe = Sequence.size(); p != pe; ++p) {
750750 const MaskRolPair &P = Sequence[p];
751 OS << format("{ 0x%08X, %2u }, ", P.Mask, P.RotateLeft);
752 }
753 OS << "{ 0, 0 }";
751 printMask(OS << "{ ", P.Mask);
752 OS << format(", %2u }, ", P.RotateLeft);
753 }
754 OS << "{ LaneBitmask::getNone(), 0 }";
754755 if (s+1 != se)
755756 OS << ", ";
756757 OS << " // Sequence " << Idx << "\n";
773774 " const {\n"
774775 " --IdxA; assert(IdxA < " << SubRegIndices.size()
775776 << " && \"Subregister index out of bounds\");\n"
776 " LaneBitmask Result = 0;\n"
777 " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; Ops->Mask != 0; ++Ops)"
778 " {\n"
779 " LaneBitmask Masked = LaneMask & Ops->Mask;\n"
780 " Result |= (Masked << Ops->RotateLeft) & 0xFFFFFFFF;\n"
781 " Result |= (Masked >> ((32 - Ops->RotateLeft) & 0x1F));\n"
777 " LaneBitmask Result;\n"
778 " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; !Ops->Mask.none(); ++Ops) {\n"
779 " LaneBitmask::Type M = LaneMask.getAsInteger() & Ops->Mask.getAsInteger();\n"
780 " unsigned S = Ops->RotateLeft;\n"
781 " Result |= LaneBitmask((M << S) | (M >> (LaneBitmask::BitWidth - S)));\n"
782782 " }\n"
783783 " return Result;\n"
784784 "}\n\n";
789789 " LaneMask &= getSubRegIndexLaneMask(IdxA);\n"
790790 " --IdxA; assert(IdxA < " << SubRegIndices.size()
791791 << " && \"Subregister index out of bounds\");\n"
792 " LaneBitmask Result = 0;\n"
793 " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; Ops->Mask != 0; ++Ops)"
794 " {\n"
795 " LaneBitmask Rotated = (LaneMask >> Ops->RotateLeft) |\n"
796 " ((LaneMask << ((32 - Ops->RotateLeft) & 0x1F)) & 0xFFFFFFFF);\n"
797 " Result |= Rotated & Ops->Mask;\n"
792 " LaneBitmask Result;\n"
793 " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; !Ops->Mask.none(); ++Ops) {\n"
794 " LaneBitmask::Type M = LaneMask.getAsInteger();\n"
795 " unsigned S = Ops->RotateLeft;\n"
796 " Result |= LaneBitmask((M >> S) | (M << (LaneBitmask::BitWidth - S)));\n"
798797 " }\n"
799798 " return Result;\n"
800799 "}\n\n";
893892 LaneMaskVec.insert(LaneMaskVec.begin(), RUMasks.begin(), RUMasks.end());
894893 // Terminator mask should not be used inside of the list.
895894 #ifndef NDEBUG
896 for (unsigned M : LaneMaskVec) {
897 assert(M != ~0u && "terminator mask should not be part of the list");
895 for (LaneBitmask M : LaneMaskVec) {
896 assert(!M.all() && "terminator mask should not be part of the list");
898897 }
899898 #endif
900899 LaneMaskSeqs.add(LaneMaskVec);
915914 OS << "};\n\n";
916915
917916 // Emit the shared table of regunit lane mask sequences.
918 OS << "extern const unsigned " << TargetName << "LaneMaskLists[] = {\n";
919 LaneMaskSeqs.emit(OS, printMask, "~0u");
917 OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[] = {\n";
918 LaneMaskSeqs.emit(OS, printMask, "LaneBitmask::getAll()");
920919 OS << "};\n\n";
921920
922921 // Emit the table of sub-register indexes.
11961195 OS << "\" };\n\n";
11971196
11981197 // Emit SubRegIndex lane masks, including 0.
1199 OS << "\nstatic const unsigned SubRegIndexLaneMaskTable[] = {\n ~0u,\n";
1198 OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n LaneBitmask::getAll(),\n";
12001199 for (const auto &Idx : SubRegIndices) {
1201 OS << format(" 0x%08x, // ", Idx.LaneMask) << Idx.getName() << '\n';
1200 printMask(OS << " ", Idx.LaneMask);
1201 OS << ", // " << Idx.getName() << '\n';
12021202 }
12031203 OS << " };\n\n";
12041204
13161316 << "MCRegisterClasses[" << RC.getName() << "RegClassID],\n "
13171317 << "VTLists + " << VTSeqs.get(RC.VTs) << ",\n " << RC.getName()
13181318 << "SubClassMask,\n SuperRegIdxSeqs + "
1319 << SuperRegIdxSeqs.get(SuperRegIdxLists[RC.EnumValue]) << ",\n "
1320 << format("0x%08x,\n ", RC.LaneMask)
1321 << (unsigned)RC.AllocationPriority << ",\n "
1319 << SuperRegIdxSeqs.get(SuperRegIdxLists[RC.EnumValue]) << ",\n ";
1320 printMask(OS, RC.LaneMask);
1321 OS << ",\n " << (unsigned)RC.AllocationPriority << ",\n "
13221322 << (RC.HasDisjunctSubRegs?"true":"false")
13231323 << ", /* HasDisjunctSubRegs */\n "
13241324 << (RC.CoveredBySubRegs?"true":"false")
14071407 // Emit the constructor of the class...
14081408 OS << "extern const MCRegisterDesc " << TargetName << "RegDesc[];\n";
14091409 OS << "extern const MCPhysReg " << TargetName << "RegDiffLists[];\n";
1410 OS << "extern const unsigned " << TargetName << "LaneMaskLists[];\n";
1410 OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[];\n";
14111411 OS << "extern const char " << TargetName << "RegStrings[];\n";
14121412 OS << "extern const char " << TargetName << "RegClassStrings[];\n";
14131413 OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2];\n";
14221422 << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour, unsigned PC)\n"
14231423 << " : TargetRegisterInfo(" << TargetName << "RegInfoDesc"
14241424 << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() <<",\n"
1425 << " SubRegIndexNameTable, SubRegIndexLaneMaskTable, 0x";
1426 OS.write_hex(RegBank.CoveringLanes);
1425 << " SubRegIndexNameTable, SubRegIndexLaneMaskTable, ";
1426 printMask(OS, RegBank.CoveringLanes);
14271427 OS << ") {\n"
14281428 << " InitMCRegisterInfo(" << TargetName << "RegDesc, " << Regs.size() + 1
14291429 << ", RA, PC,\n " << TargetName