llvm.org GIT mirror llvm / 327236c
Basic x86 code generation for atomic load and store instructions. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@138478 91177308-0d34-0410-b5e6-96231b3b80d8 Eli Friedman 9 years ago
10 changed file(s) with 254 addition(s) and 29 deletion(s). Raw diff Collapse all Expand all
596596 // two integer constants: an AtomicOrdering and a SynchronizationScope.
597597 ATOMIC_FENCE,
598598
599 // Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr)
600 // This corresponds to "load atomic" instruction.
601 ATOMIC_LOAD,
602
603 // OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)
604 // This corresponds to "store atomic" instruction.
605 ATOMIC_STORE,
606
599607 // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
600 // this corresponds to the atomic.lcs intrinsic.
601 // cmp is compared to *ptr, and if equal, swap is stored in *ptr.
602 // the return is always the original value in *ptr
608 // This corresponds to the cmpxchg instruction.
603609 ATOMIC_CMP_SWAP,
604610
605611 // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
606 // this corresponds to the atomic.swap intrinsic.
607 // amt is stored to *ptr atomically.
608 // the return is always the original value in *ptr
612 // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
613 // These correspond to the atomicrmw instruction.
609614 ATOMIC_SWAP,
610
611 // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
612 // this corresponds to the atomic.load.[OpName] intrinsic.
613 // op(*ptr, amt) is stored to *ptr atomically.
614 // the return is always the original value in *ptr
615615 ATOMIC_LOAD_ADD,
616616 ATOMIC_LOAD_SUB,
617617 ATOMIC_LOAD_AND,
597597 AtomicOrdering Ordering,
598598 SynchronizationScope SynchScope);
599599
600 /// getAtomic - Gets a node for an atomic op, produces result and chain and
601 /// takes 2 operands.
600 /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
601 /// and chain and takes 2 operands.
602602 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
603603 SDValue Ptr, SDValue Val, const Value* PtrVal,
604 unsigned Alignment, AtomicOrdering Ordering,
605 SynchronizationScope SynchScope);
606 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
607 SDValue Ptr, SDValue Val, MachineMemOperand *MMO,
608 AtomicOrdering Ordering,
609 SynchronizationScope SynchScope);
610
611 /// getAtomic - Gets a node for an atomic op, produces result and chain and
612 /// takes 1 operand.
613 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
614 SDValue Chain, SDValue Ptr, const Value* PtrVal,
604615 unsigned Alignment,
605616 AtomicOrdering Ordering,
606617 SynchronizationScope SynchScope);
607 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
608 SDValue Ptr, SDValue Val,
609 MachineMemOperand *MMO,
618 SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, EVT VT,
619 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
610620 AtomicOrdering Ordering,
611621 SynchronizationScope SynchScope);
612622
975975 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
976976 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
977977 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
978 N->getOpcode() == ISD::ATOMIC_LOAD ||
979 N->getOpcode() == ISD::ATOMIC_STORE ||
978980 N->isTargetMemoryOpcode();
979981 }
980982 };
10241026 InitAtomic(Ordering, SynchScope);
10251027 InitOperands(Ops, Chain, Ptr, Val);
10261028 }
1029 AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
1030 SDValue Chain, SDValue Ptr,
1031 MachineMemOperand *MMO,
1032 AtomicOrdering Ordering, SynchronizationScope SynchScope)
1033 : MemSDNode(Opc, dl, VTL, MemVT, MMO) {
1034 InitAtomic(Ordering, SynchScope);
1035 InitOperands(Ops, Chain, Ptr);
1036 }
10271037
10281038 const SDValue &getBasePtr() const { return getOperand(1); }
10291039 const SDValue &getVal() const { return getOperand(2); }
10471057 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
10481058 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
10491059 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1050 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX;
1060 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1061 N->getOpcode() == ISD::ATOMIC_LOAD ||
1062 N->getOpcode() == ISD::ATOMIC_STORE;
10511063 }
10521064 };
10531065
212212 ]>;
213213 def SDTAtomic2 : SDTypeProfile<1, 2, [
214214 SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1>
215 ]>;
216 def SDTAtomicStore : SDTypeProfile<0, 2, [
217 SDTCisPtrTy<0>, SDTCisInt<1>
218 ]>;
219 def SDTAtomicLoad : SDTypeProfile<1, 1, [
220 SDTCisInt<0>, SDTCisPtrTy<1>
215221 ]>;
216222
217223 def SDTConvertOp : SDTypeProfile<1, 5, [ //cvtss, su, us, uu, ff, fs, fu, sf, su
425431 def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", SDTAtomic2,
426432 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
427433 def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,
434 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
435 def atomic_load : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
436 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
437 def atomic_store : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
428438 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
429439
430440 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,
843853 defm atomic_load_max : binary_atomic_op;
844854 defm atomic_load_umin : binary_atomic_op;
845855 defm atomic_load_umax : binary_atomic_op;
856 defm atomic_store : binary_atomic_op;
857
858 def atomic_load_8 :
859 PatFrag<(ops node:$ptr),
860 (atomic_load node:$ptr), [{
861 return cast(N)->getMemoryVT() == MVT::i8;
862 }]>;
863 def atomic_load_16 :
864 PatFrag<(ops node:$ptr),
865 (atomic_load node:$ptr), [{
866 return cast(N)->getMemoryVT() == MVT::i16;
867 }]>;
868 def atomic_load_32 :
869 PatFrag<(ops node:$ptr),
870 (atomic_load node:$ptr), [{
871 return cast(N)->getMemoryVT() == MVT::i32;
872 }]>;
873 def atomic_load_64 :
874 PatFrag<(ops node:$ptr),
875 (atomic_load node:$ptr), [{
876 return cast(N)->getMemoryVT() == MVT::i64;
877 }]>;
846878
847879 //===----------------------------------------------------------------------===//
848880 // Selection DAG CONVERT_RNDSAT patterns
816816 case ISD::SIGN_EXTEND_INREG: {
817817 EVT InnerType = cast(Node->getOperand(1))->getVT();
818818 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
819 break;
820 }
821 case ISD::ATOMIC_STORE: {
822 Action = TLI.getOperationAction(Node->getOpcode(),
823 Node->getOperand(2).getValueType());
819824 break;
820825 }
821826 case ISD::SELECT_CC:
431431 case ISD::ATOMIC_LOAD_MIN:
432432 case ISD::ATOMIC_LOAD_MAX:
433433 case ISD::ATOMIC_LOAD_UMIN:
434 case ISD::ATOMIC_LOAD_UMAX: {
434 case ISD::ATOMIC_LOAD_UMAX:
435 case ISD::ATOMIC_LOAD:
436 case ISD::ATOMIC_STORE: {
435437 const AtomicSDNode *AT = cast(N);
436438 ID.AddInteger(AT->getMemoryVT().getRawBits());
437439 ID.AddInteger(AT->getRawSubclassData());
39033905 Opcode == ISD::ATOMIC_LOAD_MAX ||
39043906 Opcode == ISD::ATOMIC_LOAD_UMIN ||
39053907 Opcode == ISD::ATOMIC_LOAD_UMAX ||
3906 Opcode == ISD::ATOMIC_SWAP) &&
3908 Opcode == ISD::ATOMIC_SWAP ||
3909 Opcode == ISD::ATOMIC_STORE) &&
39073910 "Invalid Atomic Op");
39083911
39093912 EVT VT = Val.getValueType();
39103913
3911 SDVTList VTs = getVTList(VT, MVT::Other);
3914 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
3915 getVTList(VT, MVT::Other);
39123916 FoldingSetNodeID ID;
39133917 ID.AddInteger(MemVT.getRawBits());
39143918 SDValue Ops[] = {Chain, Ptr, Val};
39213925 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
39223926 Ptr, Val, MMO,
39233927 Ordering, SynchScope);
3928 CSEMap.InsertNode(N, IP);
3929 AllNodes.push_back(N);
3930 return SDValue(N, 0);
3931 }
3932
3933 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
3934 EVT VT, SDValue Chain,
3935 SDValue Ptr,
3936 const Value* PtrVal,
3937 unsigned Alignment,
3938 AtomicOrdering Ordering,
3939 SynchronizationScope SynchScope) {
3940 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3941 Alignment = getEVTAlignment(MemVT);
3942
3943 MachineFunction &MF = getMachineFunction();
3944 unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
3945
3946 // For now, atomics are considered to be volatile always.
3947 Flags |= MachineMemOperand::MOVolatile;
3948
3949 MachineMemOperand *MMO =
3950 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
3951 MemVT.getStoreSize(), Alignment);
3952
3953 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO,
3954 Ordering, SynchScope);
3955 }
3956
3957 SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
3958 EVT VT, SDValue Chain,
3959 SDValue Ptr,
3960 MachineMemOperand *MMO,
3961 AtomicOrdering Ordering,
3962 SynchronizationScope SynchScope) {
3963 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
3964
3965 SDVTList VTs = getVTList(VT, MVT::Other);
3966 FoldingSetNodeID ID;
3967 ID.AddInteger(MemVT.getRawBits());
3968 SDValue Ops[] = {Chain, Ptr};
3969 AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
3970 void* IP = 0;
3971 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
3972 cast(E)->refineAlignment(MMO);
3973 return SDValue(E, 0);
3974 }
3975 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
3976 Ptr, MMO, Ordering, SynchScope);
39243977 CSEMap.InsertNode(N, IP);
39253978 AllNodes.push_back(N);
39263979 return SDValue(N, 0);
57945847 case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax";
57955848 case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin";
57965849 case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax";
5850 case ISD::ATOMIC_LOAD: return "AtomicLoad";
5851 case ISD::ATOMIC_STORE: return "AtomicStore";
57975852 case ISD::PCMARKER: return "PCMarker";
57985853 case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
57995854 case ISD::SRCVALUE: return "SrcValue";
31483148 }
31493149
31503150 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3151 if (I.isAtomic())
3152 return visitAtomicLoad(I);
3153
31513154 const Value *SV = I.getOperand(0);
31523155 SDValue Ptr = getValue(SV);
31533156
32253228 }
32263229
32273230 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3231 if (I.isAtomic())
3232 return visitAtomicStore(I);
3233
32283234 const Value *SrcV = I.getOperand(0);
32293235 const Value *PtrV = I.getOperand(1);
32303236
32763282 }
32773283
32783284 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
3285 SynchronizationScope Scope,
32793286 bool Before, DebugLoc dl,
32803287 SelectionDAG &DAG,
32813288 const TargetLowering &TLI) {
32933300 }
32943301 SDValue Ops[3];
32953302 Ops[0] = Chain;
3296 Ops[1] = DAG.getConstant(SequentiallyConsistent, TLI.getPointerTy());
3297 Ops[2] = DAG.getConstant(Order, TLI.getPointerTy());
3303 Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
3304 Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
32983305 return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);
32993306 }
33003307
33013308 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
33023309 DebugLoc dl = getCurDebugLoc();
33033310 AtomicOrdering Order = I.getOrdering();
3311 SynchronizationScope Scope = I.getSynchScope();
33043312
33053313 SDValue InChain = getRoot();
33063314
33073315 if (TLI.getInsertFencesForAtomic())
3308 InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);
3316 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
3317 DAG, TLI);
33093318
33103319 SDValue L =
33113320 DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
33153324 getValue(I.getCompareOperand()),
33163325 getValue(I.getNewValOperand()),
33173326 MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
3318 I.getOrdering(), I.getSynchScope());
3327 TLI.getInsertFencesForAtomic() ? Monotonic : Order,
3328 Scope);
33193329
33203330 SDValue OutChain = L.getValue(1);
33213331
33223332 if (TLI.getInsertFencesForAtomic())
3323 OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);
3333 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
3334 DAG, TLI);
33243335
33253336 setValue(&I, L);
33263337 DAG.setRoot(OutChain);
33443355 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
33453356 }
33463357 AtomicOrdering Order = I.getOrdering();
3358 SynchronizationScope Scope = I.getSynchScope();
33473359
33483360 SDValue InChain = getRoot();
33493361
33503362 if (TLI.getInsertFencesForAtomic())
3351 InChain = InsertFenceForAtomic(InChain, Order, true, dl, DAG, TLI);
3363 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
3364 DAG, TLI);
33523365
33533366 SDValue L =
33543367 DAG.getAtomic(NT, dl,
33583371 getValue(I.getValOperand()),
33593372 I.getPointerOperand(), 0 /* Alignment */,
33603373 TLI.getInsertFencesForAtomic() ? Monotonic : Order,
3361 I.getSynchScope());
3374 Scope);
33623375
33633376 SDValue OutChain = L.getValue(1);
33643377
33653378 if (TLI.getInsertFencesForAtomic())
3366 OutChain = InsertFenceForAtomic(OutChain, Order, false, dl, DAG, TLI);
3379 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
3380 DAG, TLI);
33673381
33683382 setValue(&I, L);
33693383 DAG.setRoot(OutChain);
33763390 Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
33773391 Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
33783392 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));
3393 }
3394
3395 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
3396 DebugLoc dl = getCurDebugLoc();
3397 AtomicOrdering Order = I.getOrdering();
3398 SynchronizationScope Scope = I.getSynchScope();
3399
3400 SDValue InChain = getRoot();
3401
3402 if (TLI.getInsertFencesForAtomic())
3403 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
3404 DAG, TLI);
3405
3406 EVT VT = EVT::getEVT(I.getType());
3407
3408 SDValue L =
3409 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3410 getValue(I.getPointerOperand()),
3411 I.getPointerOperand(), I.getAlignment(),
3412 TLI.getInsertFencesForAtomic() ? Monotonic : Order,
3413 Scope);
3414
3415 SDValue OutChain = L.getValue(1);
3416
3417 if (TLI.getInsertFencesForAtomic())
3418 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
3419 DAG, TLI);
3420
3421 setValue(&I, L);
3422 DAG.setRoot(OutChain);
3423 }
3424
3425 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
3426 DebugLoc dl = getCurDebugLoc();
3427
3428 AtomicOrdering Order = I.getOrdering();
3429 SynchronizationScope Scope = I.getSynchScope();
3430
3431 SDValue InChain = getRoot();
3432
3433 if (TLI.getInsertFencesForAtomic())
3434 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
3435 DAG, TLI);
3436
3437 SDValue OutChain =
3438 DAG.getAtomic(ISD::ATOMIC_STORE, dl,
3439 getValue(I.getValueOperand()).getValueType().getSimpleVT(),
3440 InChain,
3441 getValue(I.getPointerOperand()),
3442 getValue(I.getValueOperand()),
3443 I.getPointerOperand(), I.getAlignment(),
3444 TLI.getInsertFencesForAtomic() ? Monotonic : Order,
3445 Scope);
3446
3447 if (TLI.getInsertFencesForAtomic())
3448 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
3449 DAG, TLI);
3450
3451 DAG.setRoot(OutChain);
33793452 }
33803453
33813454 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
525525 void visitPHI(const PHINode &I);
526526 void visitCall(const CallInst &I);
527527 bool visitMemCmpCall(const CallInst &I);
528
528 void visitAtomicLoad(const LoadInst &I);
529 void visitAtomicStore(const StoreInst &I);
530
529531 void visitInlineAsm(ImmutableCallSite CS);
530532 const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
531533 void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
463463 MVT VT = IntVTs[i];
464464 setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom);
465465 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
466 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
466467 }
467468
468469 if (!Subtarget->is64Bit()) {
99989999 cast(Node)->getSynchScope());
999910000 }
1000010001
10002 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
10003 SDNode *Node = Op.getNode();
10004 DebugLoc dl = Node->getDebugLoc();
10005
10006 // Convert seq_cst store -> xchg
10007 if (cast(Node)->getOrdering() == SequentiallyConsistent) {
10008 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
10009 cast(Node)->getMemoryVT(),
10010 Node->getOperand(0),
10011 Node->getOperand(1), Node->getOperand(2),
10012 cast(Node)->getSrcValue(),
10013 cast(Node)->getAlignment(),
10014 cast(Node)->getOrdering(),
10015 cast(Node)->getSynchScope());
10016 return Swap.getValue(1);
10017 }
10018 // Other atomic stores have a simple pattern.
10019 return Op;
10020 }
10021
1000110022 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
1000210023 EVT VT = Op.getNode()->getValueType(0);
1000310024
1003410055 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op,DAG);
1003510056 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG);
1003610057 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
10058 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG);
1003710059 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
1003810060 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1003910061 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
16901690 (AND64ri8 GR64:$src1, i64immSExt8:$src2)>;
16911691 def : Pat<(and GR64:$src1, i64immSExt32:$src2),
16921692 (AND64ri32 GR64:$src1, i64immSExt32:$src2)>;
1693
1694 def : Pat<(atomic_load_8 addr:$src), (MOV8rm addr:$src)>;
1695 def : Pat<(atomic_load_16 addr:$src), (MOV16rm addr:$src)>;
1696 def : Pat<(atomic_load_32 addr:$src), (MOV32rm addr:$src)>;
1697 def : Pat<(atomic_load_64 addr:$src), (MOV64rm addr:$src)>;
1698
1699 def : Pat<(atomic_store_8 addr:$ptr, GR8:$val),
1700 (MOV8mr addr:$ptr, GR8:$val)>;
1701 def : Pat<(atomic_store_16 addr:$ptr, GR16:$val),
1702 (MOV16mr addr:$ptr, GR16:$val)>;
1703 def : Pat<(atomic_store_32 addr:$ptr, GR32:$val),
1704 (MOV32mr addr:$ptr, GR32:$val)>;
1705 def : Pat<(atomic_store_64 addr:$ptr, GR64:$val),
1706 (MOV64mr addr:$ptr, GR64:$val)>;