llvm.org GIT mirror llvm / a7e2c08
[MachineMemOperand] Move synchronization scope and atomic orderings from SDNode to MachineMemOperand, and remove redundant getAtomic* member functions from SelectionDAG. Differential Revision: https://reviews.llvm.org/D24577 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@284312 91177308-0d34-0410-b5e6-96231b3b80d8 Konstantin Zhuravlyov 4 years ago
12 changed file(s) with 149 addition(s) and 146 deletion(s). Raw diff Collapse all Expand all
566566 /// getMachineMemOperand - Allocate a new MachineMemOperand.
567567 /// MachineMemOperands are owned by the MachineFunction and need not be
568568 /// explicitly deallocated.
569 MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo,
570 MachineMemOperand::Flags f,
571 uint64_t s, unsigned base_alignment,
572 const AAMDNodes &AAInfo = AAMDNodes(),
573 const MDNode *Ranges = nullptr);
569 MachineMemOperand *getMachineMemOperand(
570 MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
571 unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
572 const MDNode *Ranges = nullptr,
573 SynchronizationScope SynchScope = CrossThread,
574 AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
575 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
574576
575577 /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
576578 /// an existing one, adjusting by an offset and using the given size.
1818 #include "llvm/ADT/BitmaskEnum.h"
1919 #include "llvm/ADT/PointerUnion.h"
2020 #include "llvm/CodeGen/PseudoSourceValue.h"
21 #include "llvm/IR/Instructions.h"
2122 #include "llvm/IR/Metadata.h"
2223 #include "llvm/IR/Value.h" // PointerLikeTypeTraits
24 #include "llvm/Support/AtomicOrdering.h"
2325 #include "llvm/Support/DataTypes.h"
2426
2527 namespace llvm {
114116 };
115117
116118 private:
119 /// Atomic information for this memory operation.
120 struct MachineAtomicInfo {
121 /// Synchronization scope for this memory operation.
122 unsigned SynchScope : 1; // enum SynchronizationScope
123 /// Atomic ordering requirements for this memory operation. For cmpxchg
124 /// atomic operations, atomic ordering requirements when store occurs.
125 unsigned Ordering : 4; // enum AtomicOrdering
126 /// For cmpxchg atomic operations, atomic ordering requirements when store
127 /// does not occur.
128 unsigned FailureOrdering : 4; // enum AtomicOrdering
129 };
130
117131 MachinePointerInfo PtrInfo;
118132 uint64_t Size;
119133 Flags FlagVals;
120134 uint16_t BaseAlignLog2; // log_2(base_alignment) + 1
135 MachineAtomicInfo AtomicInfo;
121136 AAMDNodes AAInfo;
122137 const MDNode *Ranges;
123138
124139 public:
125140 /// Construct a MachineMemOperand object with the specified PtrInfo, flags,
126 /// size, and base alignment.
141 /// size, and base alignment. For atomic operations the synchronization scope
142 /// and atomic ordering requirements must also be specified. For cmpxchg
143 /// atomic operations the atomic ordering requirements when store does not
144 /// occur must also be specified.
127145 MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s,
128146 unsigned base_alignment,
129147 const AAMDNodes &AAInfo = AAMDNodes(),
130 const MDNode *Ranges = nullptr);
148 const MDNode *Ranges = nullptr,
149 SynchronizationScope SynchScope = CrossThread,
150 AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
151 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
131152
132153 const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
133154
175196 /// Return the range tag for the memory reference.
176197 const MDNode *getRanges() const { return Ranges; }
177198
199 /// Return the synchronization scope for this memory operation.
200 SynchronizationScope getSynchScope() const {
201 return static_cast(AtomicInfo.SynchScope);
202 }
203
204 /// Return the atomic ordering requirements for this memory operation.
205 AtomicOrdering getOrdering() const {
206 return static_cast(AtomicInfo.Ordering);
207 }
208
209 /// For cmpxchg atomic operations, return the atomic ordering requirements
210 /// when store occurs.
211 AtomicOrdering getSuccessOrdering() const {
212 return getOrdering();
213 }
214
215 /// For cmpxchg atomic operations, return the atomic ordering requirements
216 /// when store does not occur.
217 AtomicOrdering getFailureOrdering() const {
218 return static_cast(AtomicInfo.FailureOrdering);
219 }
220
178221 bool isLoad() const { return FlagVals & MOLoad; }
179222 bool isStore() const { return FlagVals & MOStore; }
180223 bool isVolatile() const { return FlagVals & MOVolatile; }
181224 bool isNonTemporal() const { return FlagVals & MONonTemporal; }
182225 bool isDereferenceable() const { return FlagVals & MODereferenceable; }
183226 bool isInvariant() const { return FlagVals & MOInvariant; }
227
228 /// Returns true if this operation has an atomic ordering requirement of
229 /// unordered or higher, false otherwise.
230 bool isAtomic() const { return getOrdering() != AtomicOrdering::NotAtomic; }
184231
185232 /// Returns true if this memory operation doesn't have any ordering
186233 /// constraints other than normal aliasing. Volatile and atomic memory
855855 SynchronizationScope SynchScope);
856856 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
857857 SDVTList VTs, SDValue Chain, SDValue Ptr,
858 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
859 AtomicOrdering SuccessOrdering,
860 AtomicOrdering FailureOrdering,
861 SynchronizationScope SynchScope);
858 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
862859
863860 /// Gets a node for an atomic op, produces result (if relevant)
864861 /// and chain and takes 2 operands.
867864 unsigned Alignment, AtomicOrdering Ordering,
868865 SynchronizationScope SynchScope);
869866 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
870 SDValue Ptr, SDValue Val, MachineMemOperand *MMO,
871 AtomicOrdering Ordering, SynchronizationScope SynchScope);
867 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
872868
873869 /// Gets a node for an atomic op, produces result and chain and
874870 /// takes 1 operand.
875871 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
876 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
877 AtomicOrdering Ordering, SynchronizationScope SynchScope);
872 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
878873
879874 /// Gets a node for an atomic op, produces result and chain and takes N
880875 /// operands.
881876 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
882877 SDVTList VTList, ArrayRef Ops,
883 MachineMemOperand *MMO, AtomicOrdering SuccessOrdering,
884 AtomicOrdering FailureOrdering,
885 SynchronizationScope SynchScope);
886 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
887 SDVTList VTList, ArrayRef Ops,
888 MachineMemOperand *MMO, AtomicOrdering Ordering,
889 SynchronizationScope SynchScope);
878 MachineMemOperand *MMO);
890879
891880 /// Creates a MemIntrinsicNode that may produce a
892881 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
423423 uint16_t IsNonTemporal : 1;
424424 uint16_t IsDereferenceable : 1;
425425 uint16_t IsInvariant : 1;
426 uint16_t SynchScope : 1; // enum SynchronizationScope
427 uint16_t Ordering : 4; // enum AtomicOrdering
428426 };
429 enum { NumMemSDNodeBits = NumSDNodeBits + 9 };
427 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
430428
431429 class LSBaseSDNodeBitfields {
432430 friend class LSBaseSDNode;
11121110 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
11131111 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
11141112
1115 AtomicOrdering getOrdering() const {
1116 return static_cast(MemSDNodeBits.Ordering);
1117 }
1118 SynchronizationScope getSynchScope() const {
1119 return static_cast(MemSDNodeBits.SynchScope);
1120 }
1121
11221113 // Returns the offset from the location of the access.
11231114 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
11241115
11271118
11281119 /// Returns the Ranges that describes the dereference.
11291120 const MDNode *getRanges() const { return MMO->getRanges(); }
1121
1122 /// Return the synchronization scope for this memory operation.
1123 SynchronizationScope getSynchScope() const { return MMO->getSynchScope(); }
1124
1125 /// Return the atomic ordering requirements for this memory operation.
1126 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
11301127
11311128 /// Return the type of the in-memory value.
11321129 EVT getMemoryVT() const { return MemoryVT; }
11901187
11911188 /// This is an SDNode representing atomic operations.
11921189 class AtomicSDNode : public MemSDNode {
1193 /// For cmpxchg instructions, the ordering requirements when a store does not
1194 /// occur.
1195 AtomicOrdering FailureOrdering;
1196
1197 void InitAtomic(AtomicOrdering SuccessOrdering,
1198 AtomicOrdering FailureOrdering,
1199 SynchronizationScope SynchScope) {
1200 MemSDNodeBits.Ordering = static_cast(SuccessOrdering);
1201 assert(getOrdering() == SuccessOrdering && "Value truncated");
1202 MemSDNodeBits.SynchScope = static_cast(SynchScope);
1203 assert(getSynchScope() == SynchScope && "Value truncated");
1204 this->FailureOrdering = FailureOrdering;
1205 }
1206
12071190 public:
12081191 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1209 EVT MemVT, MachineMemOperand *MMO,
1210 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
1211 SynchronizationScope SynchScope)
1212 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1213 InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
1214 }
1192 EVT MemVT, MachineMemOperand *MMO)
1193 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {}
12151194
12161195 const SDValue &getBasePtr() const { return getOperand(1); }
12171196 const SDValue &getVal() const { return getOperand(2); }
12181197
1219 AtomicOrdering getSuccessOrdering() const {
1220 return getOrdering();
1221 }
1222
1223 // Not quite enough room in SubclassData for everything, so failure gets its
1224 // own field.
1225 AtomicOrdering getFailureOrdering() const {
1226 return FailureOrdering;
1227 }
1228
1198 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1199 /// otherwise.
12291200 bool isCompareAndSwap() const {
12301201 unsigned Op = getOpcode();
1231 return Op == ISD::ATOMIC_CMP_SWAP || Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1202 return Op == ISD::ATOMIC_CMP_SWAP ||
1203 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1204 }
1205
1206 /// For cmpxchg atomic operations, return the atomic ordering requirements
1207 /// when store occurs.
1208 AtomicOrdering getSuccessOrdering() const {
1209 assert(isCompareAndSwap() && "Must be cmpxchg operation");
1210 return MMO->getSuccessOrdering();
1211 }
1212
1213 /// For cmpxchg atomic operations, return the atomic ordering requirements
1214 /// when store does not occur.
1215 AtomicOrdering getFailureOrdering() const {
1216 assert(isCompareAndSwap() && "Must be cmpxchg operation");
1217 return MMO->getFailureOrdering();
12321218 }
12331219
12341220 // Methods to support isa and dyn_cast
305305
306306 MachineMemOperand *MachineFunction::getMachineMemOperand(
307307 MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
308 unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges) {
308 unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
309 SynchronizationScope SynchScope, AtomicOrdering Ordering,
310 AtomicOrdering FailureOrdering) {
309311 return new (Allocator)
310 MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges);
312 MachineMemOperand(PtrInfo, f, s, base_alignment, AAInfo, Ranges,
313 SynchScope, Ordering, FailureOrdering);
311314 }
312315
313316 MachineMemOperand *
317320 return new (Allocator)
318321 MachineMemOperand(MachinePointerInfo(MMO->getValue(),
319322 MMO->getOffset()+Offset),
320 MMO->getFlags(), Size,
321 MMO->getBaseAlignment());
323 MMO->getFlags(), Size, MMO->getBaseAlignment(),
324 AAMDNodes(), nullptr, MMO->getSynchScope(),
325 MMO->getOrdering(), MMO->getFailureOrdering());
322326 return new (Allocator)
323327 MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(),
324328 MMO->getOffset()+Offset),
325 MMO->getFlags(), Size,
326 MMO->getBaseAlignment());
329 MMO->getFlags(), Size, MMO->getBaseAlignment(),
330 AAMDNodes(), nullptr, MMO->getSynchScope(),
331 MMO->getOrdering(), MMO->getFailureOrdering());
327332 }
328333
329334 MachineInstr::mmo_iterator
354359 getMachineMemOperand((*I)->getPointerInfo(),
355360 (*I)->getFlags() & ~MachineMemOperand::MOStore,
356361 (*I)->getSize(), (*I)->getBaseAlignment(),
357 (*I)->getAAInfo());
362 (*I)->getAAInfo(), nullptr,
363 (*I)->getSynchScope(), (*I)->getOrdering(),
364 (*I)->getFailureOrdering());
358365 Result[Index] = JustLoad;
359366 }
360367 ++Index;
386393 getMachineMemOperand((*I)->getPointerInfo(),
387394 (*I)->getFlags() & ~MachineMemOperand::MOLoad,
388395 (*I)->getSize(), (*I)->getBaseAlignment(),
389 (*I)->getAAInfo());
396 (*I)->getAAInfo(), nullptr,
397 (*I)->getSynchScope(), (*I)->getOrdering(),
398 (*I)->getFailureOrdering());
390399 Result[Index] = JustStore;
391400 }
392401 ++Index;
536536 MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
537537 uint64_t s, unsigned int a,
538538 const AAMDNodes &AAInfo,
539 const MDNode *Ranges)
539 const MDNode *Ranges,
540 SynchronizationScope SynchScope,
541 AtomicOrdering Ordering,
542 AtomicOrdering FailureOrdering)
540543 : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1),
541544 AAInfo(AAInfo), Ranges(Ranges) {
542545 assert((PtrInfo.V.isNull() || PtrInfo.V.is() ||
544547 "invalid pointer value");
545548 assert(getBaseAlignment() == a && "Alignment is not a power of 2!");
546549 assert((isLoad() || isStore()) && "Not a load/store!");
550
551 AtomicInfo.SynchScope = static_cast(SynchScope);
552 assert(getSynchScope() == SynchScope && "Value truncated");
553 AtomicInfo.Ordering = static_cast(Ordering);
554 assert(getOrdering() == Ordering && "Value truncated");
555 AtomicInfo.FailureOrdering = static_cast(FailureOrdering);
556 assert(getFailureOrdering() == FailureOrdering && "Value truncated");
547557 }
548558
549559 /// Profile - Gather unique data for the object.
28342834 SDValue Swap = DAG.getAtomicCmpSwap(
28352835 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
28362836 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2837 cast(Node)->getMemOperand(),
2838 cast(Node)->getOrdering(),
2839 cast(Node)->getOrdering(),
2840 cast(Node)->getSynchScope());
2837 cast(Node)->getMemOperand());
28412838 Results.push_back(Swap.getValue(0));
28422839 Results.push_back(Swap.getValue(1));
28432840 break;
28482845 cast(Node)->getMemoryVT(),
28492846 Node->getOperand(0),
28502847 Node->getOperand(1), Node->getOperand(2),
2851 cast(Node)->getMemOperand(),
2852 cast(Node)->getOrdering(),
2853 cast(Node)->getSynchScope());
2848 cast(Node)->getMemOperand());
28542849 Results.push_back(Swap.getValue(1));
28552850 break;
28562851 }
28622857 SDValue Res = DAG.getAtomicCmpSwap(
28632858 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
28642859 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2865 Node->getOperand(3), cast(Node)->getMemOperand(),
2866 cast(Node)->getSuccessOrdering(),
2867 cast(Node)->getFailureOrdering(),
2868 cast(Node)->getSynchScope());
2860 Node->getOperand(3), cast(Node)->getMemOperand());
28692861
28702862 SDValue ExtRes = Res;
28712863 SDValue LHS = Res;
187187 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
188188 N->getMemoryVT(), ResVT,
189189 N->getChain(), N->getBasePtr(),
190 N->getMemOperand(), N->getOrdering(),
191 N->getSynchScope());
190 N->getMemOperand());
192191 // Legalize the chain result - switch anything that used the old chain to
193192 // use the new one.
194193 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
200199 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
201200 N->getMemoryVT(),
202201 N->getChain(), N->getBasePtr(),
203 Op2, N->getMemOperand(), N->getOrdering(),
204 N->getSynchScope());
202 Op2, N->getMemOperand());
205203 // Legalize the chain result - switch anything that used the old chain to
206204 // use the new one.
207205 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
224222 SDValue Res = DAG.getAtomicCmpSwap(
225223 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
226224 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
227 N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(),
228 N->getSynchScope());
225 N->getMemOperand());
229226 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
230227 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
231228 return Res.getValue(1);
237234 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
238235 SDValue Res = DAG.getAtomicCmpSwap(
239236 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
240 N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(),
241 N->getFailureOrdering(), N->getSynchScope());
237 N->getBasePtr(), Op2, Op3, N->getMemOperand());
242238 // Update the use to N with the newly created Res.
243239 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
244240 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
996992 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
997993 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
998994 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
999 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
1000 N->getOrdering(), N->getSynchScope());
995 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1001996 }
1002997
1003998 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
13671362 SDValue Tmp = DAG.getAtomicCmpSwap(
13681363 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
13691364 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1370 AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(),
1371 AN->getSynchScope());
1365 AN->getMemOperand());
13721366
13731367 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
13741368 // success simply by comparing the loaded value against the ingoing
27322726 SDValue Swap = DAG.getAtomicCmpSwap(
27332727 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
27342728 cast(N)->getMemoryVT(), VTs, N->getOperand(0),
2735 N->getOperand(1), Zero, Zero, cast(N)->getMemOperand(),
2736 cast(N)->getOrdering(),
2737 cast(N)->getOrdering(),
2738 cast(N)->getSynchScope());
2729 N->getOperand(1), Zero, Zero, cast(N)->getMemOperand());
27392730
27402731 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
27412732 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
32233214 cast(N)->getMemoryVT(),
32243215 N->getOperand(0),
32253216 N->getOperand(1), N->getOperand(2),
3226 cast(N)->getMemOperand(),
3227 cast(N)->getOrdering(),
3228 cast(N)->getSynchScope());
3217 cast(N)->getMemOperand());
32293218 return Swap.getValue(1);
32303219 }
32313220
48304830
48314831 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
48324832 SDVTList VTList, ArrayRef Ops,
4833 MachineMemOperand *MMO,
4834 AtomicOrdering SuccessOrdering,
4835 AtomicOrdering FailureOrdering,
4836 SynchronizationScope SynchScope) {
4833 MachineMemOperand *MMO) {
48374834 FoldingSetNodeID ID;
48384835 ID.AddInteger(MemVT.getRawBits());
48394836 AddNodeIDNode(ID, Opcode, VTList, Ops);
48454842 }
48464843
48474844 auto *N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(),
4848 VTList, MemVT, MMO, SuccessOrdering,
4849 FailureOrdering, SynchScope);
4845 VTList, MemVT, MMO);
48504846 createOperands(N, Ops);
48514847
48524848 CSEMap.InsertNode(N, IP);
48534849 InsertNode(N);
48544850 return SDValue(N, 0);
4855 }
4856
4857 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
4858 SDVTList VTList, ArrayRef Ops,
4859 MachineMemOperand *MMO, AtomicOrdering Ordering,
4860 SynchronizationScope SynchScope) {
4861 return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
4862 Ordering, SynchScope);
48634851 }
48644852
48654853 SDValue SelectionDAG::getAtomicCmpSwap(
48814869 auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
48824870 MachineMemOperand::MOStore;
48834871 MachineMemOperand *MMO =
4884 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
4885
4886 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
4887 SuccessOrdering, FailureOrdering, SynchScope);
4872 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
4873 AAMDNodes(), nullptr, SynchScope, SuccessOrdering,
4874 FailureOrdering);
4875
4876 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
48884877 }
48894878
48904879 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
48914880 EVT MemVT, SDVTList VTs, SDValue Chain,
48924881 SDValue Ptr, SDValue Cmp, SDValue Swp,
4893 MachineMemOperand *MMO,
4894 AtomicOrdering SuccessOrdering,
4895 AtomicOrdering FailureOrdering,
4896 SynchronizationScope SynchScope) {
4882 MachineMemOperand *MMO) {
48974883 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
48984884 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
48994885 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
49004886
49014887 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4902 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
4903 SuccessOrdering, FailureOrdering, SynchScope);
4888 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
49044889 }
49054890
49064891 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
49264911
49274912 MachineMemOperand *MMO =
49284913 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4929 MemVT.getStoreSize(), Alignment);
4930
4931 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4932 Ordering, SynchScope);
4914 MemVT.getStoreSize(), Alignment, AAMDNodes(),
4915 nullptr, SynchScope, Ordering);
4916
4917 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
49334918 }
49344919
49354920 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
49364921 SDValue Chain, SDValue Ptr, SDValue Val,
4937 MachineMemOperand *MMO, AtomicOrdering Ordering,
4938 SynchronizationScope SynchScope) {
4922 MachineMemOperand *MMO) {
49394923 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
49404924 Opcode == ISD::ATOMIC_LOAD_SUB ||
49414925 Opcode == ISD::ATOMIC_LOAD_AND ||
49554939 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
49564940 getVTList(VT, MVT::Other);
49574941 SDValue Ops[] = {Chain, Ptr, Val};
4958 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4942 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
49594943 }
49604944
49614945 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
49624946 EVT VT, SDValue Chain, SDValue Ptr,
4963 MachineMemOperand *MMO, AtomicOrdering Ordering,
4964 SynchronizationScope SynchScope) {
4947 MachineMemOperand *MMO) {
49654948 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
49664949
49674950 SDVTList VTs = getVTList(VT, MVT::Other);
49684951 SDValue Ops[] = {Chain, Ptr};
4969 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4952 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
49704953 }
49714954
49724955 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
39723972 MachineMemOperand::MOLoad,
39733973 VT.getStoreSize(),
39743974 I.getAlignment() ? I.getAlignment() :
3975 DAG.getEVTAlignment(VT));
3975 DAG.getEVTAlignment(VT),
3976 AAMDNodes(), nullptr, Scope, Order);
39763977
39773978 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
39783979 SDValue L =
39793980 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3980 getValue(I.getPointerOperand()), MMO,
3981 Order, Scope);
3981 getValue(I.getPointerOperand()), MMO);
39823982
39833983 SDValue OutChain = L.getValue(1);
39843984
33093309 if (NegSrc2.getNode())
33103310 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
33113311 Node->getChain(), Node->getBasePtr(), NegSrc2,
3312 Node->getMemOperand(), Node->getOrdering(),
3313 Node->getSynchScope());
3312 Node->getMemOperand());
33143313
33153314 // Use the node as-is.
33163315 return Op;
2159621596 AtomicSDNode *AN = cast(N.getNode());
2159721597 RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
2159821598 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,
21599 RHS, AN->getMemOperand(), AN->getOrdering(),
21600 AN->getSynchScope());
21599 RHS, AN->getMemOperand());
2160121600 }
2160221601 assert(Opc == ISD::ATOMIC_LOAD_ADD &&
2160321602 "Used AtomicRMW ops other than Add should have been expanded!");
2162821627 cast(Node)->getMemoryVT(),
2162921628 Node->getOperand(0),
2163021629 Node->getOperand(1), Node->getOperand(2),
21631 cast(Node)->getMemOperand(),
21632 cast(Node)->getOrdering(),
21633 cast(Node)->getSynchScope());
21630 cast(Node)->getMemOperand());
2163421631 return Swap.getValue(1);
2163521632 }
2163621633 // Other atomic stores have a simple pattern.