llvm.org GIT mirror llvm / ca396e3
IR: add a second ordering operand to cmpxhg for failure The syntax for "cmpxchg" should now look something like: cmpxchg i32* %addr, i32 42, i32 3 acquire monotonic where the second ordering argument gives the required semantics in the case that no exchange takes place. It should be no stronger than the first ordering constraint and cannot be either "release" or "acq_rel" (since no store will have taken place). rdar://problem/15996804 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@203559 91177308-0d34-0410-b5e6-96231b3b80d8 Tim Northover 5 years ago
60 changed file(s) with 513 addition(s) and 312 deletion(s). Raw diff Collapse all Expand all
14951495 Atomic instructions (:ref:`cmpxchg `,
14961496 :ref:`atomicrmw `, :ref:`fence `,
14971497 :ref:`atomic load `, and :ref:`atomic store `) take
1498 an ordering parameter that determines which other atomic instructions on
1498 ordering parameters that determine which other atomic instructions on
14991499 the same address they *synchronize with*. These semantics are borrowed
15001500 from Java and C++0x, but are somewhat more colloquial. If these
15011501 descriptions aren't precise enough, check those specs (see spec
49894989
49904990 ::
49914991
4992 cmpxchg [volatile] * , , [singlethread] <ordering> ; yields {ty}
4992 cmpxchg [volatile] * , , [singlethread] <success ordering> ; yields {ty}
49934993
49944994 Overview:
49954995 """""""""
50125012 to modify the number or order of execution of this ``cmpxchg`` with
50135013 other :ref:`volatile operations `.
50145014
5015 The :ref:`ordering ` argument specifies how this ``cmpxchg``
5016 synchronizes with other atomic operations.
5015 The success and failure :ref:`ordering ` arguments specify how this
5016 ``cmpxchg`` synchronizes with other atomic operations. The both ordering
5017 parameters must be at least ``monotonic``, the ordering constraint on failure
5018 must be no stronger than that on success, and the failure ordering cannot be
5019 either ``release`` or ``acq_rel``.
50175020
50185021 The optional "``singlethread``" argument declares that the ``cmpxchg``
50195022 is only atomic with respect to code (usually signal handlers) running in
50315034 equal, '````' is written. The original value at the location is
50325035 returned.
50335036
5034 A successful ``cmpxchg`` is a read-modify-write instruction for the purpose
5035 of identifying release sequences. A failed ``cmpxchg`` is equivalent to an
5036 atomic load with an ordering parameter determined by dropping any
5037 ``release`` part of the ``cmpxchg``'s ordering.
5037 A successful ``cmpxchg`` is a read-modify-write instruction for the purpose of
5038 identifying release sequences. A failed ``cmpxchg`` is equivalent to an atomic
5039 load with an ordering parameter determined the second ordering parameter.
50385040
50395041 Example:
50405042 """"""""
50485050 loop:
50495051 %cmp = phi i32 [ %orig, %entry ], [%old, %loop]
50505052 %squared = mul i32 %cmp, %cmp
5051 %old = cmpxchg i32* %ptr, i32 %cmp, i32 %squared ; yields {i32}
5053 %old = cmpxchg i32* %ptr, i32 %cmp, i32 %squared acq_rel monotonic ; yields {i32}
50525054 %success = icmp eq i32 %cmp, %old
50535055 br i1 %success, label %done, label %loop
50545056
691691 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
692692 SDValue Ptr, SDValue Cmp, SDValue Swp,
693693 MachinePointerInfo PtrInfo, unsigned Alignment,
694 AtomicOrdering Ordering,
694 AtomicOrdering SuccessOrdering,
695 AtomicOrdering FailureOrdering,
695696 SynchronizationScope SynchScope);
696697 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
697698 SDValue Ptr, SDValue Cmp, SDValue Swp,
698699 MachineMemOperand *MMO,
699 AtomicOrdering Ordering,
700 AtomicOrdering SuccessOrdering,
701 AtomicOrdering FailureOrdering,
700702 SynchronizationScope SynchScope);
701703
702704 /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
725727 /// getAtomic - Gets a node for an atomic op, produces result and chain and
726728 /// takes N operands.
727729 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList,
728 SDValue* Ops, unsigned NumOps, MachineMemOperand *MMO,
729 AtomicOrdering Ordering,
730 SDValue *Ops, unsigned NumOps, MachineMemOperand *MMO,
731 AtomicOrdering SuccessOrdering,
732 AtomicOrdering FailureOrdering,
730733 SynchronizationScope SynchScope);
734 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList,
735 SDValue *Ops, unsigned NumOps, MachineMemOperand *MMO,
736 AtomicOrdering Ordering, SynchronizationScope SynchScope);
731737
732738 /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
733739 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
10921092 class AtomicSDNode : public MemSDNode {
10931093 SDUse Ops[4];
10941094
1095 void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
1095 /// For cmpxchg instructions, the ordering requirements when a store does not
1096 /// occur.
1097 AtomicOrdering FailureOrdering;
1098
1099 void InitAtomic(AtomicOrdering SuccessOrdering,
1100 AtomicOrdering FailureOrdering,
1101 SynchronizationScope SynchScope) {
10961102 // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
1097 assert((Ordering & 15) == Ordering &&
1103 assert((SuccessOrdering & 15) == SuccessOrdering &&
1104 "Ordering may not require more than 4 bits!");
1105 assert((FailureOrdering & 15) == FailureOrdering &&
10981106 "Ordering may not require more than 4 bits!");
10991107 assert((SynchScope & 1) == SynchScope &&
11001108 "SynchScope may not require more than 1 bit!");
1101 SubclassData |= Ordering << 8;
1109 SubclassData |= SuccessOrdering << 8;
11021110 SubclassData |= SynchScope << 12;
1103 assert(getOrdering() == Ordering && "Ordering encoding error!");
1111 this->FailureOrdering = FailureOrdering;
1112 assert(getSuccessOrdering() == SuccessOrdering &&
1113 "Ordering encoding error!");
1114 assert(getFailureOrdering() == FailureOrdering &&
1115 "Ordering encoding error!");
11041116 assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
11051117 }
11061118
11141126 // SrcVal: address to update as a Value (used for MemOperand)
11151127 // Align: alignment of memory
11161128 AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
1117 EVT MemVT,
1118 SDValue Chain, SDValue Ptr,
1119 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
1120 AtomicOrdering Ordering, SynchronizationScope SynchScope)
1121 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1122 InitAtomic(Ordering, SynchScope);
1129 EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
1130 MachineMemOperand *MMO, AtomicOrdering Ordering,
1131 SynchronizationScope SynchScope)
1132 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1133 InitAtomic(Ordering, Ordering, SynchScope);
11231134 InitOperands(Ops, Chain, Ptr, Cmp, Swp);
11241135 }
11251136 AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
11281139 SDValue Val, MachineMemOperand *MMO,
11291140 AtomicOrdering Ordering, SynchronizationScope SynchScope)
11301141 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1131 InitAtomic(Ordering, SynchScope);
1142 InitAtomic(Ordering, Ordering, SynchScope);
11321143 InitOperands(Ops, Chain, Ptr, Val);
11331144 }
11341145 AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
11371148 MachineMemOperand *MMO,
11381149 AtomicOrdering Ordering, SynchronizationScope SynchScope)
11391150 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1140 InitAtomic(Ordering, SynchScope);
1151 InitAtomic(Ordering, Ordering, SynchScope);
11411152 InitOperands(Ops, Chain, Ptr);
11421153 }
11431154 AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT,
11441155 SDValue* AllOps, SDUse *DynOps, unsigned NumOps,
11451156 MachineMemOperand *MMO,
1146 AtomicOrdering Ordering, SynchronizationScope SynchScope)
1157 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
1158 SynchronizationScope SynchScope)
11471159 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1148 InitAtomic(Ordering, SynchScope);
1160 InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
11491161 assert((DynOps || NumOps <= array_lengthof(Ops)) &&
11501162 "Too many ops for internal storage!");
11511163 InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps);
11531165
11541166 const SDValue &getBasePtr() const { return getOperand(1); }
11551167 const SDValue &getVal() const { return getOperand(2); }
1168
1169 AtomicOrdering getSuccessOrdering() const {
1170 return getOrdering();
1171 }
1172
1173 // Not quite enough room in SubclassData for everything, so failure gets its
1174 // own field.
1175 AtomicOrdering getFailureOrdering() const {
1176 return FailureOrdering;
1177 }
11561178
11571179 bool isCompareAndSwap() const {
11581180 unsigned Op = getOpcode();
933933 const Twine &Name = "") {
934934 return Insert(new FenceInst(Context, Ordering, SynchScope), Name);
935935 }
936 AtomicCmpXchgInst *CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
937 AtomicOrdering Ordering,
938 SynchronizationScope SynchScope = CrossThread) {
939 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope));
936 AtomicCmpXchgInst *
937 CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
938 AtomicOrdering SuccessOrdering,
939 AtomicOrdering FailureOrdering,
940 SynchronizationScope SynchScope = CrossThread) {
941 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering,
942 FailureOrdering, SynchScope));
940943 }
941944 AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
942945 AtomicOrdering Ordering,
462462 class AtomicCmpXchgInst : public Instruction {
463463 void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
464464 void Init(Value *Ptr, Value *Cmp, Value *NewVal,
465 AtomicOrdering Ordering, SynchronizationScope SynchScope);
465 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
466 SynchronizationScope SynchScope);
466467 protected:
467468 AtomicCmpXchgInst *clone_impl() const override;
468469 public:
471472 return User::operator new(s, 3);
472473 }
473474 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
474 AtomicOrdering Ordering, SynchronizationScope SynchScope,
475 AtomicOrdering SuccessOrdering,
476 AtomicOrdering FailureOrdering,
477 SynchronizationScope SynchScope,
475478 Instruction *InsertBefore = 0);
476479 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
477 AtomicOrdering Ordering, SynchronizationScope SynchScope,
480 AtomicOrdering SuccessOrdering,
481 AtomicOrdering FailureOrdering,
482 SynchronizationScope SynchScope,
478483 BasicBlock *InsertAtEnd);
479484
480485 /// isVolatile - Return true if this is a cmpxchg from a volatile memory
495500 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
496501
497502 /// Set the ordering constraint on this cmpxchg.
498 void setOrdering(AtomicOrdering Ordering) {
503 void setSuccessOrdering(AtomicOrdering Ordering) {
499504 assert(Ordering != NotAtomic &&
500505 "CmpXchg instructions can only be atomic.");
501 setInstructionSubclassData((getSubclassDataFromInstruction() & 3) |
506 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x1c) |
502507 (Ordering << 2));
508 }
509
510 void setFailureOrdering(AtomicOrdering Ordering) {
511 assert(Ordering != NotAtomic &&
512 "CmpXchg instructions can only be atomic.");
513 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0xe0) |
514 (Ordering << 5));
503515 }
504516
505517 /// Specify whether this cmpxchg is atomic and orders other operations with
511523 }
512524
513525 /// Returns the ordering constraint on this cmpxchg.
514 AtomicOrdering getOrdering() const {
515 return AtomicOrdering(getSubclassDataFromInstruction() >> 2);
526 AtomicOrdering getSuccessOrdering() const {
527 return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
528 }
529
530 /// Returns the ordering constraint on this cmpxchg.
531 AtomicOrdering getFailureOrdering() const {
532 return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7);
516533 }
517534
518535 /// Returns whether this cmpxchg is atomic between threads or only within a
534551 /// \brief Returns the address space of the pointer operand.
535552 unsigned getPointerAddressSpace() const {
536553 return getPointerOperand()->getType()->getPointerAddressSpace();
554 }
555
556 /// \brief Returns the strongest permitted ordering on failure, given the
557 /// desired ordering on success.
558 ///
559 /// If the comparison in a cmpxchg operation fails, there is no atomic store
560 /// so release semantics cannot be provided. So this function drops explicit
561 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
562 /// operation would remain SequentiallyConsistent.
563 static AtomicOrdering
564 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
565 switch (SuccessOrdering) {
566 default: llvm_unreachable("invalid cmpxchg success ordering");
567 case Release:
568 case Monotonic:
569 return Monotonic;
570 case AcquireRelease:
571 case Acquire:
572 return Acquire;
573 case SequentiallyConsistent:
574 return SequentiallyConsistent;
575 }
537576 }
538577
539578 // Methods for support type inquiry through isa, cast, and dyn_cast:
337337 AliasAnalysis::ModRefResult
338338 AliasAnalysis::getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc) {
339339 // Acquire/Release cmpxchg has properties that matter for arbitrary addresses.
340 if (CX->getOrdering() > Monotonic)
340 if (CX->getSuccessOrdering() > Monotonic)
341341 return ModRef;
342342
343343 // If the cmpxchg address does not alias the location, it does not access it.
15171517 Scope = CrossThread;
15181518 if (EatIfPresent(lltok::kw_singlethread))
15191519 Scope = SingleThread;
1520
1521 return ParseOrdering(Ordering);
1522 }
1523
1524 /// ParseOrdering
1525 /// ::= AtomicOrdering
1526 ///
1527 /// This sets Ordering to the parsed value.
1528 bool LLParser::ParseOrdering(AtomicOrdering &Ordering) {
15201529 switch (Lex.getKind()) {
15211530 default: return TokError("Expected ordering on atomic instruction");
15221531 case lltok::kw_unordered: Ordering = Unordered; break;
41924201
41934202 /// ParseCmpXchg
41944203 /// ::= 'cmpxchg' 'volatile'? TypeAndValue ',' TypeAndValue ',' TypeAndValue
4195 /// 'singlethread'? AtomicOrdering
4204 /// 'singlethread'? AtomicOrdering AtomicOrdering
41964205 int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
41974206 Value *Ptr, *Cmp, *New; LocTy PtrLoc, CmpLoc, NewLoc;
41984207 bool AteExtraComma = false;
4199 AtomicOrdering Ordering = NotAtomic;
4208 AtomicOrdering SuccessOrdering = NotAtomic;
4209 AtomicOrdering FailureOrdering = NotAtomic;
42004210 SynchronizationScope Scope = CrossThread;
42014211 bool isVolatile = false;
42024212
42084218 ParseTypeAndValue(Cmp, CmpLoc, PFS) ||
42094219 ParseToken(lltok::comma, "expected ',' after cmpxchg cmp operand") ||
42104220 ParseTypeAndValue(New, NewLoc, PFS) ||
4211 ParseScopeAndOrdering(true /*Always atomic*/, Scope, Ordering))
4221 ParseScopeAndOrdering(true /*Always atomic*/, Scope, SuccessOrdering) ||
4222 ParseOrdering(FailureOrdering))
42124223 return true;
42134224
4214 if (Ordering == Unordered)
4225 if (SuccessOrdering == Unordered || FailureOrdering == Unordered)
42154226 return TokError("cmpxchg cannot be unordered");
4227 if (SuccessOrdering < FailureOrdering)
4228 return TokError("cmpxchg must be at least as ordered on success as failure");
4229 if (FailureOrdering == Release || FailureOrdering == AcquireRelease)
4230 return TokError("cmpxchg failure ordering cannot include release semantics");
42164231 if (!Ptr->getType()->isPointerTy())
42174232 return Error(PtrLoc, "cmpxchg operand must be a pointer");
42184233 if (cast(Ptr->getType())->getElementType() != Cmp->getType())
42264241 return Error(NewLoc, "cmpxchg operand must be power-of-two byte-sized"
42274242 " integer");
42284243
4229 AtomicCmpXchgInst *CXI =
4230 new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, Scope);
4244 AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering,
4245 FailureOrdering, Scope);
42314246 CXI->setVolatile(isVolatile);
42324247 Inst = CXI;
42334248 return AteExtraComma ? InstExtraComma : InstNormal;
208208 bool ParseOptionalAlignment(unsigned &Alignment);
209209 bool ParseScopeAndOrdering(bool isAtomic, SynchronizationScope &Scope,
210210 AtomicOrdering &Ordering);
211 bool ParseOrdering(AtomicOrdering &Ordering);
211212 bool ParseOptionalStackAlignment(unsigned &Alignment);
212213 bool ParseOptionalCommaAlign(unsigned &Alignment, bool &AteExtraComma);
213214 bool ParseOptionalCommaInAlloca(bool &IsInAlloca);
28812881 break;
28822882 }
28832883 case bitc::FUNC_CODE_INST_CMPXCHG: {
2884 // CMPXCHG:[ptrty, ptr, cmp, new, vol, ordering, synchscope]
2884 // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope,
2885 // failureordering]
28852886 unsigned OpNum = 0;
28862887 Value *Ptr, *Cmp, *New;
28872888 if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
28892890 cast(Ptr->getType())->getElementType(), Cmp) ||
28902891 popValue(Record, OpNum, NextValueNo,
28912892 cast(Ptr->getType())->getElementType(), New) ||
2892 OpNum+3 != Record.size())
2893 return Error(InvalidRecord);
2894 AtomicOrdering Ordering = GetDecodedOrdering(Record[OpNum+1]);
2895 if (Ordering == NotAtomic || Ordering == Unordered)
2893 (OpNum + 3 != Record.size() && OpNum + 4 != Record.size()))
2894 return Error(InvalidRecord);
2895 AtomicOrdering SuccessOrdering = GetDecodedOrdering(Record[OpNum+1]);
2896 if (SuccessOrdering == NotAtomic || SuccessOrdering == Unordered)
28962897 return Error(InvalidRecord);
28972898 SynchronizationScope SynchScope = GetDecodedSynchScope(Record[OpNum+2]);
2898 I = new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope);
2899
2900 AtomicOrdering FailureOrdering;
2901 if (Record.size() < 7)
2902 FailureOrdering =
2903 AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering);
2904 else
2905 FailureOrdering = GetDecodedOrdering(Record[OpNum+3]);
2906
2907 I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering,
2908 SynchScope);
28992909 cast(I)->setVolatile(Record[OpNum]);
29002910 InstructionList.push_back(I);
29012911 break;
14401440 pushValue(I.getOperand(2), InstID, Vals, VE); // newval.
14411441 Vals.push_back(cast(I).isVolatile());
14421442 Vals.push_back(GetEncodedOrdering(
1443 cast(I).getOrdering()));
1443 cast(I).getSuccessOrdering()));
14441444 Vals.push_back(GetEncodedSynchScope(
14451445 cast(I).getSynchScope()));
1446 Vals.push_back(GetEncodedOrdering(
1447 cast(I).getFailureOrdering()));
14461448 break;
14471449 case Instruction::AtomicRMW:
14481450 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
28772877 Node->getOperand(1), Zero, Zero,
28782878 cast(Node)->getMemOperand(),
28792879 cast(Node)->getOrdering(),
2880 cast(Node)->getOrdering(),
28802881 cast(Node)->getSynchScope());
28812882 Results.push_back(Swap.getValue(0));
28822883 Results.push_back(Swap.getValue(1));
192192 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
193193 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
194194 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
195 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
196 N->getMemoryVT(), N->getChain(), N->getBasePtr(),
197 Op2, Op3, N->getMemOperand(), N->getOrdering(),
198 N->getSynchScope());
195 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
196 N->getChain(), N->getBasePtr(), Op2, Op3,
197 N->getMemOperand(), N->getSuccessOrdering(),
198 N->getFailureOrdering(), N->getSynchScope());
199199 // Legalized the chain result - switch anything that used the old chain to
200200 // use the new one.
201201 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
24472447 N->getOperand(1), Zero, Zero,
24482448 cast(N)->getMemOperand(),
24492449 cast(N)->getOrdering(),
2450 cast(N)->getOrdering(),
24502451 cast(N)->getSynchScope());
24512452 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
24522453 ReplaceValueWith(SDValue(N, 1), Swap.getValue(1));
42224222 }
42234223
42244224 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4225 SDVTList VTList, SDValue* Ops, unsigned NumOps,
4225 SDVTList VTList, SDValue *Ops, unsigned NumOps,
42264226 MachineMemOperand *MMO,
4227 AtomicOrdering Ordering,
4227 AtomicOrdering SuccessOrdering,
4228 AtomicOrdering FailureOrdering,
42284229 SynchronizationScope SynchScope) {
42294230 FoldingSetNodeID ID;
42304231 ID.AddInteger(MemVT.getRawBits());
42464247 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
42474248 dl.getDebugLoc(), VTList, MemVT,
42484249 Ops, DynOps, NumOps, MMO,
4249 Ordering, SynchScope);
4250 SuccessOrdering, FailureOrdering,
4251 SynchScope);
42504252 CSEMap.InsertNode(N, IP);
42514253 AllNodes.push_back(N);
42524254 return SDValue(N, 0);
4255 }
4256
4257 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4258 SDVTList VTList, SDValue *Ops, unsigned NumOps,
4259 MachineMemOperand *MMO,
4260 AtomicOrdering Ordering,
4261 SynchronizationScope SynchScope) {
4262 return getAtomic(Opcode, dl, MemVT, VTList, Ops, NumOps, MMO, Ordering,
4263 Ordering, SynchScope);
42534264 }
42544265
42554266 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
42564267 SDValue Chain, SDValue Ptr, SDValue Cmp,
42574268 SDValue Swp, MachinePointerInfo PtrInfo,
42584269 unsigned Alignment,
4259 AtomicOrdering Ordering,
4270 AtomicOrdering SuccessOrdering,
4271 AtomicOrdering FailureOrdering,
42604272 SynchronizationScope SynchScope) {
42614273 if (Alignment == 0) // Ensure that codegen never sees alignment 0
42624274 Alignment = getEVTAlignment(MemVT);
42774289 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
42784290
42794291 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
4280 Ordering, SynchScope);
4292 SuccessOrdering, FailureOrdering, SynchScope);
42814293 }
42824294
42834295 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
42844296 SDValue Chain,
42854297 SDValue Ptr, SDValue Cmp,
42864298 SDValue Swp, MachineMemOperand *MMO,
4287 AtomicOrdering Ordering,
4299 AtomicOrdering SuccessOrdering,
4300 AtomicOrdering FailureOrdering,
42884301 SynchronizationScope SynchScope) {
42894302 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
42904303 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
42934306
42944307 SDVTList VTs = getVTList(VT, MVT::Other);
42954308 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4296 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope);
4309 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, SuccessOrdering,
4310 FailureOrdering, SynchScope);
42974311 }
42984312
42994313 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
36043604
36053605 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
36063606 SDLoc dl = getCurSDLoc();
3607 AtomicOrdering Order = I.getOrdering();
3607 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3608 AtomicOrdering FailureOrder = I.getFailureOrdering();
36083609 SynchronizationScope Scope = I.getSynchScope();
36093610
36103611 SDValue InChain = getRoot();
36113612
36123613 const TargetLowering *TLI = TM.getTargetLowering();
36133614 if (TLI->getInsertFencesForAtomic())
3614 InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
3615 InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
36153616 DAG, *TLI);
36163617
36173618 SDValue L =
36223623 getValue(I.getCompareOperand()),
36233624 getValue(I.getNewValOperand()),
36243625 MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
3625 TLI->getInsertFencesForAtomic() ? Monotonic : Order,
3626 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
3627 TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder,
36263628 Scope);
36273629
36283630 SDValue OutChain = L.getValue(1);
36293631
36303632 if (TLI->getInsertFencesForAtomic())
3631 OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
3633 OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
36323634 DAG, *TLI);
36333635
36343636 setValue(&I, L);
12251225 }
12261226 }
12271227
1228 void AssemblyWriter::writeAtomicCmpXchg(AtomicOrdering SuccessOrdering,
1229 AtomicOrdering FailureOrdering,
1230 SynchronizationScope SynchScope) {
1231 assert(SuccessOrdering != NotAtomic && FailureOrdering != NotAtomic);
1232
1233 switch (SynchScope) {
1234 case SingleThread: Out << " singlethread"; break;
1235 case CrossThread: break;
1236 }
1237
1238 switch (SuccessOrdering) {
1239 default: Out << " "; break;
1240 case Unordered: Out << " unordered"; break;
1241 case Monotonic: Out << " monotonic"; break;
1242 case Acquire: Out << " acquire"; break;
1243 case Release: Out << " release"; break;
1244 case AcquireRelease: Out << " acq_rel"; break;
1245 case SequentiallyConsistent: Out << " seq_cst"; break;
1246 }
1247
1248 switch (FailureOrdering) {
1249 default: Out << " "; break;
1250 case Unordered: Out << " unordered"; break;
1251 case Monotonic: Out << " monotonic"; break;
1252 case Acquire: Out << " acquire"; break;
1253 case Release: Out << " release"; break;
1254 case AcquireRelease: Out << " acq_rel"; break;
1255 case SequentiallyConsistent: Out << " seq_cst"; break;
1256 }
1257 }
1258
12281259 void AssemblyWriter::writeParamOperand(const Value *Operand,
12291260 AttributeSet Attrs, unsigned Idx) {
12301261 if (Operand == 0) {
20172048 if (SI->getAlignment())
20182049 Out << ", align " << SI->getAlignment();
20192050 } else if (const AtomicCmpXchgInst *CXI = dyn_cast(&I)) {
2020 writeAtomic(CXI->getOrdering(), CXI->getSynchScope());
2051 writeAtomicCmpXchg(CXI->getSuccessOrdering(), CXI->getFailureOrdering(),
2052 CXI->getSynchScope());
20212053 } else if (const AtomicRMWInst *RMWI = dyn_cast(&I)) {
20222054 writeAtomic(RMWI->getOrdering(), RMWI->getSynchScope());
20232055 } else if (const FenceInst *FI = dyn_cast(&I)) {
8989 void writeOperand(const Value *Op, bool PrintType);
9090 void writeParamOperand(const Value *Operand, AttributeSet Attrs,unsigned Idx);
9191 void writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope);
92 void writeAtomicCmpXchg(AtomicOrdering SuccessOrdering,
93 AtomicOrdering FailureOrdering,
94 SynchronizationScope SynchScope);
9295
9396 void writeAllMDNodes();
9497 void writeMDNode(unsigned Slot, const MDNode *Node);
312312 FI->getSynchScope() == cast(FI)->getSynchScope();
313313 if (const AtomicCmpXchgInst *CXI = dyn_cast(this))
314314 return CXI->isVolatile() == cast(I)->isVolatile() &&
315 CXI->getOrdering() == cast(I)->getOrdering() &&
315 CXI->getSuccessOrdering() ==
316 cast(I)->getSuccessOrdering() &&
317 CXI->getFailureOrdering() ==
318 cast(I)->getFailureOrdering() &&
316319 CXI->getSynchScope() == cast(I)->getSynchScope();
317320 if (const AtomicRMWInst *RMWI = dyn_cast(this))
318321 return RMWI->getOperation() == cast(I)->getOperation() &&
383386 FI->getSynchScope() == cast(I)->getSynchScope();
384387 if (const AtomicCmpXchgInst *CXI = dyn_cast(this))
385388 return CXI->isVolatile() == cast(I)->isVolatile() &&
386 CXI->getOrdering() == cast(I)->getOrdering() &&
389 CXI->getSuccessOrdering() ==
390 cast(I)->getSuccessOrdering() &&
391 CXI->getFailureOrdering() ==
392 cast(I)->getFailureOrdering() &&
387393 CXI->getSynchScope() == cast(I)->getSynchScope();
388394 if (const AtomicRMWInst *RMWI = dyn_cast(this))
389395 return RMWI->getOperation() == cast(I)->getOperation() &&
12151215 //===----------------------------------------------------------------------===//
12161216
12171217 void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal,
1218 AtomicOrdering Ordering,
1218 AtomicOrdering SuccessOrdering,
1219 AtomicOrdering FailureOrdering,
12191220 SynchronizationScope SynchScope) {
12201221 Op<0>() = Ptr;
12211222 Op<1>() = Cmp;
12221223 Op<2>() = NewVal;
1223 setOrdering(Ordering);
1224 setSuccessOrdering(SuccessOrdering);
1225 setFailureOrdering(FailureOrdering);
12241226 setSynchScope(SynchScope);
12251227
12261228 assert(getOperand(0) && getOperand(1) && getOperand(2) &&
12331235 assert(getOperand(2)->getType() ==
12341236 cast(getOperand(0)->getType())->getElementType()
12351237 && "Ptr must be a pointer to NewVal type!");
1236 assert(Ordering != NotAtomic &&
1238 assert(SuccessOrdering != NotAtomic &&
12371239 "AtomicCmpXchg instructions must be atomic!");
1240 assert(FailureOrdering != NotAtomic &&
1241 "AtomicCmpXchg instructions must be atomic!");
1242 assert(SuccessOrdering >= FailureOrdering &&
1243 "AtomicCmpXchg success ordering must be at least as strong as fail");
1244 assert(FailureOrdering != Release && FailureOrdering != AcquireRelease &&
1245 "AtomicCmpXchg failure ordering cannot include release semantics");
12381246 }
12391247
12401248 AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
1241 AtomicOrdering Ordering,
1249 AtomicOrdering SuccessOrdering,
1250 AtomicOrdering FailureOrdering,
12421251 SynchronizationScope SynchScope,
12431252 Instruction *InsertBefore)
12441253 : Instruction(Cmp->getType(), AtomicCmpXchg,
12451254 OperandTraits::op_begin(this),
12461255 OperandTraits::operands(this),
12471256 InsertBefore) {
1248 Init(Ptr, Cmp, NewVal, Ordering, SynchScope);
1257 Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope);
12491258 }
12501259
12511260 AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
1252 AtomicOrdering Ordering,
1261 AtomicOrdering SuccessOrdering,
1262 AtomicOrdering FailureOrdering,
12531263 SynchronizationScope SynchScope,
12541264 BasicBlock *InsertAtEnd)
12551265 : Instruction(Cmp->getType(), AtomicCmpXchg,
12561266 OperandTraits::op_begin(this),
12571267 OperandTraits::operands(this),
12581268 InsertAtEnd) {
1259 Init(Ptr, Cmp, NewVal, Ordering, SynchScope);
1269 Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope);
12601270 }
12611271
12621272 //===----------------------------------------------------------------------===//
35953605 AtomicCmpXchgInst *AtomicCmpXchgInst::clone_impl() const {
35963606 AtomicCmpXchgInst *Result =
35973607 new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2),
3598 getOrdering(), getSynchScope());
3608 getSuccessOrdering(), getFailureOrdering(),
3609 getSynchScope());
35993610 Result->setVolatile(isVolatile());
36003611 return Result;
36013612 }
18281828 }
18291829
18301830 void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) {
1831 Assert1(CXI.getOrdering() != NotAtomic,
1831
1832 // FIXME: more conditions???
1833 Assert1(CXI.getSuccessOrdering() != NotAtomic,
18321834 "cmpxchg instructions must be atomic.", &CXI);
1833 Assert1(CXI.getOrdering() != Unordered,
1835 Assert1(CXI.getFailureOrdering() != NotAtomic,
1836 "cmpxchg instructions must be atomic.", &CXI);
1837 Assert1(CXI.getSuccessOrdering() != Unordered,
18341838 "cmpxchg instructions cannot be unordered.", &CXI);
1839 Assert1(CXI.getFailureOrdering() != Unordered,
1840 "cmpxchg instructions cannot be unordered.", &CXI);
1841 Assert1(CXI.getSuccessOrdering() >= CXI.getFailureOrdering(),
1842 "cmpxchg instructions be at least as constrained on success as fail",
1843 &CXI);
1844 Assert1(CXI.getFailureOrdering() != Release &&
1845 CXI.getFailureOrdering() != AcquireRelease,
1846 "cmpxchg failure ordering cannot include release semantics", &CXI);
1847
18351848 PointerType *PTy = dyn_cast(CXI.getOperand(0)->getType());
18361849 Assert1(PTy, "First cmpxchg operand must be a pointer.", &CXI);
18371850 Type *ElTy = PTy->getElementType();
60536053 Node->getOperand(i), DAG.getIntPtrConstant(1)));
60546054 }
60556055 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6056 SDValue Result =
6057 DAG.getAtomic(Node->getOpcode(), dl, MVT::i64, Tys, Ops.data(), Ops.size(),
6058 cast(Node)->getMemOperand(), AN->getOrdering(),
6059 AN->getSynchScope());
6056 SDValue Result = DAG.getAtomic(
6057 Node->getOpcode(), dl, MVT::i64, Tys, Ops.data(), Ops.size(),
6058 cast(Node)->getMemOperand(), AN->getSuccessOrdering(),
6059 AN->getFailureOrdering(), AN->getSynchScope());
60606060 SDValue OpsF[] = { Result.getValue(0), Result.getValue(1) };
60616061 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
60626062 Results.push_back(Result.getValue(2));
15661566 }
15671567 case Instruction::AtomicCmpXchg: {
15681568 const AtomicCmpXchgInst *cxi = cast(I);
1569 StringRef Ordering = ConvertAtomicOrdering(cxi->getOrdering());
1569 StringRef SuccessOrdering =
1570 ConvertAtomicOrdering(cxi->getSuccessOrdering());
1571 StringRef FailureOrdering =
1572 ConvertAtomicOrdering(cxi->getFailureOrdering());
15701573 StringRef CrossThread = ConvertAtomicSynchScope(cxi->getSynchScope());
15711574 Out << "AtomicCmpXchgInst* " << iName
15721575 << " = new AtomicCmpXchgInst("
15731576 << opNames[0] << ", " << opNames[1] << ", " << opNames[2] << ", "
1574 << Ordering << ", " << CrossThread << ", " << bbname
1577 << SuccessOrdering << ", " << FailureOrdering << ", "
1578 << CrossThread << ", " << bbname
15751579 << ");";
15761580 nl(Out) << iName << "->setName(\"";
15771581 printEscapedString(cxi->getName());
1380313803 Node->getOperand(0),
1380413804 Node->getOperand(1), Zero, Zero,
1380513805 cast(Node)->getMemOperand(),
13806 cast(Node)->getOrdering(),
1380613807 cast(Node)->getOrdering(),
1380713808 cast(Node)->getSynchScope());
1380813809 Results.push_back(Swap.getValue(0));
340340 FI->getSynchScope() == cast(I2)->getSynchScope();
341341 if (const AtomicCmpXchgInst *CXI = dyn_cast(I1))
342342 return CXI->isVolatile() == cast(I2)->isVolatile() &&
343 CXI->getOrdering() == cast(I2)->getOrdering() &&
343 CXI->getSuccessOrdering() ==
344 cast(I2)->getSuccessOrdering() &&
345 CXI->getFailureOrdering() ==
346 cast(I2)->getFailureOrdering() &&
344347 CXI->getSynchScope() == cast(I2)->getSynchScope();
345348 if (const AtomicRMWInst *RMWI = dyn_cast(I1))
346349 return RMWI->getOperation() == cast(I2)->getOperation() &&
10871087
10881088 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
10891089 handleCASOrRMW(I);
1090 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1090 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
10911091 }
10921092
10931093 // Vector manipulation.
440440 case Acquire: v = 2; break;
441441 case Release: v = 3; break;
442442 case AcquireRelease: v = 4; break;
443 case SequentiallyConsistent: v = 5; break;
444 }
445 return IRB->getInt32(v);
446 }
447
448 static ConstantInt *createFailOrdering(IRBuilder<> *IRB, AtomicOrdering ord) {
449 uint32_t v = 0;
450 switch (ord) {
451 case NotAtomic: assert(false);
452 case Unordered: // Fall-through.
453 case Monotonic: v = 0; break;
454 // case Consume: v = 1; break; // Not specified yet.
455 case Acquire: v = 2; break;
456 case Release: v = 0; break;
457 case AcquireRelease: v = 2; break;
458443 case SequentiallyConsistent: v = 5; break;
459444 }
460445 return IRB->getInt32(v);
555540 Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
556541 IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false),
557542 IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false),
558 createOrdering(&IRB, CASI->getOrdering()),
559 createFailOrdering(&IRB, CASI->getOrdering())};
543 createOrdering(&IRB, CASI->getSuccessOrdering()),
544 createOrdering(&IRB, CASI->getFailureOrdering())};
560545 CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef(Args));
561546 ReplaceInstWithInst(I, C);
562547 } else if (FenceInst *FI = dyn_cast(I)) {
99 store atomic i32 3, i32* %x release, align 4
1010 ; CHECK: store atomic volatile i32 3, i32* %x singlethread monotonic, align 4
1111 store atomic volatile i32 3, i32* %x singlethread monotonic, align 4
12 ; CHECK: cmpxchg i32* %x, i32 1, i32 0 singlethread monotonic
13 cmpxchg i32* %x, i32 1, i32 0 singlethread monotonic
14 ; CHECK: cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel
15 cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel
12 ; CHECK: cmpxchg i32* %x, i32 1, i32 0 singlethread monotonic monotonic
13 cmpxchg i32* %x, i32 1, i32 0 singlethread monotonic monotonic
14 ; CHECK: cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel acquire
15 cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel acquire
16 ; CHECK: cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic
17 cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic
1618 ; CHECK: atomicrmw add i32* %x, i32 10 seq_cst
1719 atomicrmw add i32* %x, i32 10 seq_cst
1820 ; CHECK: atomicrmw volatile xchg i32* %x, i32 10 monotonic
0 ; RUN: llvm-dis < %s.bc | FileCheck %s
1
2 ; cmpxchg-upgrade.ll.bc was produced by running a version of llvm-as from just
3 ; before the IR change on this file.
4
5 define void @test(i32* %addr) {
6 cmpxchg i32* %addr, i32 42, i32 0 monotonic
7 ; CHECK: cmpxchg i32* %addr, i32 42, i32 0 monotonic monotonic
8
9 cmpxchg i32* %addr, i32 42, i32 0 acquire
10 ; CHECK: cmpxchg i32* %addr, i32 42, i32 0 acquire acquire
11
12 cmpxchg i32* %addr, i32 42, i32 0 release
13 ; CHECK: cmpxchg i32* %addr, i32 42, i32 0 release monotonic
14
15 cmpxchg i32* %addr, i32 42, i32 0 acq_rel
16 ; CHECK: cmpxchg i32* %addr, i32 42, i32 0 acq_rel acquire
17
18 cmpxchg i32* %addr, i32 42, i32 0 seq_cst
19 ; CHECK: cmpxchg i32* %addr, i32 42, i32 0 seq_cst seq_cst
20
21 ret void
22 }
222222 entry:
223223 ;cmpxchg [volatile] * , , [singlethread]
224224
225 ; CHECK: %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic
226 %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic
227
228 ; CHECK-NEXT: %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic
229 %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic
230
231 ; CHECK-NEXT: %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic
232 %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic
233
234 ; CHECK-NEXT: %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic
235 %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic
236
237
238 ; CHECK-NEXT: %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire
239 %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire
240
241 ; CHECK-NEXT: %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire
242 %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire
243
244 ; CHECK-NEXT: %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire
245 %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire
246
247 ; CHECK-NEXT: %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire
248 %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire
249
250
251 ; CHECK-NEXT: %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release
252 %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release
253
254 ; CHECK-NEXT: %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release
255 %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release
256
257 ; CHECK-NEXT: %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release
258 %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release
259
260 ; CHECK-NEXT: %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release
261 %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release
262
263
264 ; CHECK-NEXT: %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel
265 %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel
266
267 ; CHECK-NEXT: %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel
268 %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel
269
270 ; CHECK-NEXT: %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel
271 %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel
272
273 ; CHECK-NEXT: %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel
274 %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel
275
276
277 ; CHECK-NEXT: %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst
278 %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst
279
280 ; CHECK-NEXT: %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst
281 %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst
282
283 ; CHECK-NEXT: %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst
284 %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst
285
286 ; CHECK-NEXT: %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst
287 %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst
225 ; CHECK: %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
226 %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
227
228 ; CHECK-NEXT: %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
229 %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
230
231 ; CHECK-NEXT: %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
232 %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
233
234 ; CHECK-NEXT: %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
235 %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
236
237
238 ; CHECK-NEXT: %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
239 %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
240
241 ; CHECK-NEXT: %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
242 %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
243
244 ; CHECK-NEXT: %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
245 %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
246
247 ; CHECK-NEXT: %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
248 %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
249
250
251 ; CHECK-NEXT: %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
252 %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
253
254 ; CHECK-NEXT: %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
255 %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
256
257 ; CHECK-NEXT: %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
258 %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
259
260 ; CHECK-NEXT: %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
261 %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
262
263
264 ; CHECK-NEXT: %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
265 %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
266
267 ; CHECK-NEXT: %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
268 %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
269
270 ; CHECK-NEXT: %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
271 %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
272
273 ; CHECK-NEXT: %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
274 %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
275
276
277 ; CHECK-NEXT: %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
278 %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
279
280 ; CHECK-NEXT: %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
281 %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
282
283 ; CHECK-NEXT: %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
284 %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
285
286 ; CHECK-NEXT: %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
287 %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
288288
289289 ret void
290290 }
896896
897897 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
898898 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
899 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire
899 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
900900 ; CHECK-NOT: dmb
901901 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
902902 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
919919
920920 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
921921 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
922 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
922 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
923923 ; CHECK-NOT: dmb
924924 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
925925 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
942942
943943 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
944944 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
945 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release
945 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
946946 ; CHECK-NOT: dmb
947947 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
948948 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
965965
966966 define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
967967 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
968 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic
968 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
969969 ; CHECK-NOT: dmb
970970 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
971971 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
170170 ; CHECK-THUMB: bne
171171 ; CHECK-THUMB: dmb {{ish$}}
172172
173 %r = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst
173 %r = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst seq_cst
174174 ret i64 %r
175175 }
176176
99 ; T2-LABEL: t:
1010 ; T2: ldrexb
1111 ; T2: strexb
12 %tmp0 = cmpxchg i8* %a, i8 %b, i8 %c monotonic
12 %tmp0 = cmpxchg i8* %a, i8 %b, i8 %c monotonic monotonic
1313 ret i8 %tmp0
1414 }
986986
987987 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
988988 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
989 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire
989 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
990990 ; CHECK-NOT: dmb
991991 ; CHECK-NOT: mcr
992992 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
10121012
10131013 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
10141014 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
1015 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
1015 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
10161016 ; CHECK-NOT: dmb
10171017 ; CHECK-NOT: mcr
10181018 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
10381038
10391039 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
10401040 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
1041 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release
1041 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
10421042 ; CHECK-NOT: dmb
10431043 ; CHECK-NOT: mcr
10441044 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
10641064
10651065 define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
10661066 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
1067 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic
1067 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
10681068 ; CHECK-NOT: dmb
10691069 ; CHECK-NOT: mcr
10701070 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
7676 %newval.addr = alloca i32, align 4
7777 store i32 %newval, i32* %newval.addr, align 4
7878 %tmp = load i32* %newval.addr, align 4
79 %0 = cmpxchg i32* @x, i32 %oldval, i32 %tmp monotonic
79 %0 = cmpxchg i32* @x, i32 %oldval, i32 %tmp monotonic monotonic
8080 ret i32 %0
8181
8282 ; CHECK-EL-LABEL: AtomicCmpSwap32:
332332
333333 define signext i8 @AtomicCmpSwap8(i8 signext %oldval, i8 signext %newval) nounwind {
334334 entry:
335 %0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic
335 %0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic monotonic
336336 ret i8 %0
337337
338338 ; CHECK-EL-LABEL: AtomicCmpSwap8:
428428
429429 define i32 @zeroreg() nounwind {
430430 entry:
431 %0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst
431 %0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst seq_cst
432432 %1 = icmp eq i32 %0, 1
433433 %conv = zext i1 %1 to i32
434434 ret i32 %conv
1919 %add.i = add nsw i32 %0, 2
2020 %1 = load volatile i32* %x, align 4
2121 %call1 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %add.i, i32 %1) nounwind
22 %2 = cmpxchg i32* %x, i32 1, i32 2 seq_cst
22 %2 = cmpxchg i32* %x, i32 1, i32 2 seq_cst seq_cst
2323 %3 = load volatile i32* %x, align 4
2424 %call2 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %2, i32 %3) nounwind
2525 %4 = atomicrmw xchg i32* %x, i32 1 seq_cst
528528 entry:
529529 %0 = load i8* @uc, align 1
530530 %1 = load i8* @sc, align 1
531 %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic
531 %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic
532532 store i8 %2, i8* @sc, align 1
533533 %3 = load i8* @uc, align 1
534534 %4 = load i8* @sc, align 1
535 %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic
535 %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic
536536 store i8 %5, i8* @uc, align 1
537537 %6 = load i8* @uc, align 1
538538 %7 = zext i8 %6 to i16
539539 %8 = load i8* @sc, align 1
540540 %9 = sext i8 %8 to i16
541541 %10 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
542 %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic
542 %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic
543543 store i16 %11, i16* @ss, align 2
544544 %12 = load i8* @uc, align 1
545545 %13 = zext i8 %12 to i16
546546 %14 = load i8* @sc, align 1
547547 %15 = sext i8 %14 to i16
548548 %16 = bitcast i8* bitcast (i16* @us to i8*) to i16*
549 %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic
549 %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic
550550 store i16 %17, i16* @us, align 2
551551 %18 = load i8* @uc, align 1
552552 %19 = zext i8 %18 to i32
553553 %20 = load i8* @sc, align 1
554554 %21 = sext i8 %20 to i32
555555 %22 = bitcast i8* bitcast (i32* @si to i8*) to i32*
556 %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic
556 %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic
557557 store i32 %23, i32* @si, align 4
558558 %24 = load i8* @uc, align 1
559559 %25 = zext i8 %24 to i32
560560 %26 = load i8* @sc, align 1
561561 %27 = sext i8 %26 to i32
562562 %28 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
563 %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic
563 %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic
564564 store i32 %29, i32* @ui, align 4
565565 %30 = load i8* @uc, align 1
566566 %31 = zext i8 %30 to i32
567567 %32 = load i8* @sc, align 1
568568 %33 = sext i8 %32 to i32
569569 %34 = bitcast i8* bitcast (i32* @sl to i8*) to i32*
570 %35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic
570 %35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic monotonic
571571 store i32 %35, i32* @sl, align 4
572572 %36 = load i8* @uc, align 1
573573 %37 = zext i8 %36 to i32
574574 %38 = load i8* @sc, align 1
575575 %39 = sext i8 %38 to i32
576576 %40 = bitcast i8* bitcast (i32* @ul to i8*) to i32*
577 %41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic
577 %41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic monotonic
578578 store i32 %41, i32* @ul, align 4
579579 %42 = load i8* @uc, align 1
580580 %43 = load i8* @sc, align 1
581 %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic
581 %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic
582582 %45 = icmp eq i8 %44, %42
583583 %46 = zext i1 %45 to i32
584584 store i32 %46, i32* @ui, align 4
585585 %47 = load i8* @uc, align 1
586586 %48 = load i8* @sc, align 1
587 %49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic
587 %49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic monotonic
588588 %50 = icmp eq i8 %49, %47
589589 %51 = zext i1 %50 to i32
590590 store i32 %51, i32* @ui, align 4
593593 %54 = load i8* @sc, align 1
594594 %55 = sext i8 %54 to i16
595595 %56 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
596 %57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic
596 %57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic monotonic
597597 %58 = icmp eq i16 %57, %53
598598 %59 = zext i1 %58 to i32
599599 store i32 %59, i32* @ui, align 4
602602 %62 = load i8* @sc, align 1
603603 %63 = sext i8 %62 to i16
604604 %64 = bitcast i8* bitcast (i16* @us to i8*) to i16*
605 %65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic
605 %65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic monotonic
606606 %66 = icmp eq i16 %65, %61
607607 %67 = zext i1 %66 to i32
608608 store i32 %67, i32* @ui, align 4
611611 %70 = load i8* @sc, align 1
612612 %71 = sext i8 %70 to i32
613613 %72 = bitcast i8* bitcast (i32* @si to i8*) to i32*
614 %73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic
614 %73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic monotonic
615615 %74 = icmp eq i32 %73, %69
616616 %75 = zext i1 %74 to i32
617617 store i32 %75, i32* @ui, align 4
620620 %78 = load i8* @sc, align 1
621621 %79 = sext i8 %78 to i32
622622 %80 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
623 %81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic
623 %81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic monotonic
624624 %82 = icmp eq i32 %81, %77
625625 %83 = zext i1 %82 to i32
626626 store i32 %83, i32* @ui, align 4
629629 %86 = load i8* @sc, align 1
630630 %87 = sext i8 %86 to i32
631631 %88 = bitcast i8* bitcast (i32* @sl to i8*) to i32*
632 %89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic
632 %89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic monotonic
633633 %90 = icmp eq i32 %89, %85
634634 %91 = zext i1 %90 to i32
635635 store i32 %91, i32* @ui, align 4
638638 %94 = load i8* @sc, align 1
639639 %95 = sext i8 %94 to i32
640640 %96 = bitcast i8* bitcast (i32* @ul to i8*) to i32*
641 %97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic
641 %97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic monotonic
642642 %98 = icmp eq i32 %97, %93
643643 %99 = zext i1 %98 to i32
644644 store i32 %99, i32* @ui, align 4
535535 entry:
536536 %0 = load i8* @uc, align 1
537537 %1 = load i8* @sc, align 1
538 %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic
538 %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic
539539 store i8 %2, i8* @sc, align 1
540540 %3 = load i8* @uc, align 1
541541 %4 = load i8* @sc, align 1
542 %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic
542 %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic
543543 store i8 %5, i8* @uc, align 1
544544 %6 = load i8* @uc, align 1
545545 %7 = zext i8 %6 to i16
546546 %8 = load i8* @sc, align 1
547547 %9 = sext i8 %8 to i16
548548 %10 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
549 %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic
549 %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic
550550 store i16 %11, i16* @ss, align 2
551551 %12 = load i8* @uc, align 1
552552 %13 = zext i8 %12 to i16
553553 %14 = load i8* @sc, align 1
554554 %15 = sext i8 %14 to i16
555555 %16 = bitcast i8* bitcast (i16* @us to i8*) to i16*
556 %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic
556 %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic
557557 store i16 %17, i16* @us, align 2
558558 %18 = load i8* @uc, align 1
559559 %19 = zext i8 %18 to i32
560560 %20 = load i8* @sc, align 1
561561 %21 = sext i8 %20 to i32
562562 %22 = bitcast i8* bitcast (i32* @si to i8*) to i32*
563 %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic
563 %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic
564564 store i32 %23, i32* @si, align 4
565565 %24 = load i8* @uc, align 1
566566 %25 = zext i8 %24 to i32
567567 %26 = load i8* @sc, align 1
568568 %27 = sext i8 %26 to i32
569569 %28 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
570 %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic
570 %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic
571571 store i32 %29, i32* @ui, align 4
572572 %30 = load i8* @uc, align 1
573573 %31 = zext i8 %30 to i64
574574 %32 = load i8* @sc, align 1
575575 %33 = sext i8 %32 to i64
576576 %34 = bitcast i8* bitcast (i64* @sl to i8*) to i64*
577 %35 = cmpxchg i64* %34, i64 %31, i64 %33 monotonic
577 %35 = cmpxchg i64* %34, i64 %31, i64 %33 monotonic monotonic
578578 store i64 %35, i64* @sl, align 8
579579 %36 = load i8* @uc, align 1
580580 %37 = zext i8 %36 to i64
581581 %38 = load i8* @sc, align 1
582582 %39 = sext i8 %38 to i64
583583 %40 = bitcast i8* bitcast (i64* @ul to i8*) to i64*
584 %41 = cmpxchg i64* %40, i64 %37, i64 %39 monotonic
584 %41 = cmpxchg i64* %40, i64 %37, i64 %39 monotonic monotonic
585585 store i64 %41, i64* @ul, align 8
586586 %42 = load i8* @uc, align 1
587587 %43 = load i8* @sc, align 1
588 %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic
588 %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic
589589 %45 = icmp eq i8 %44, %42
590590 %46 = zext i1 %45 to i8
591591 %47 = zext i8 %46 to i32
592592 store i32 %47, i32* @ui, align 4
593593 %48 = load i8* @uc, align 1
594594 %49 = load i8* @sc, align 1
595 %50 = cmpxchg i8* @uc, i8 %48, i8 %49 monotonic
595 %50 = cmpxchg i8* @uc, i8 %48, i8 %49 monotonic monotonic
596596 %51 = icmp eq i8 %50, %48
597597 %52 = zext i1 %51 to i8
598598 %53 = zext i8 %52 to i32
602602 %56 = load i8* @sc, align 1
603603 %57 = sext i8 %56 to i16
604604 %58 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
605 %59 = cmpxchg i16* %58, i16 %55, i16 %57 monotonic
605 %59 = cmpxchg i16* %58, i16 %55, i16 %57 monotonic monotonic
606606 %60 = icmp eq i16 %59, %55
607607 %61 = zext i1 %60 to i8
608608 %62 = zext i8 %61 to i32
612612 %65 = load i8* @sc, align 1
613613 %66 = sext i8 %65 to i16
614614 %67 = bitcast i8* bitcast (i16* @us to i8*) to i16*
615 %68 = cmpxchg i16* %67, i16 %64, i16 %66 monotonic
615 %68 = cmpxchg i16* %67, i16 %64, i16 %66 monotonic monotonic
616616 %69 = icmp eq i16 %68, %64
617617 %70 = zext i1 %69 to i8
618618 %71 = zext i8 %70 to i32
622622 %74 = load i8* @sc, align 1
623623 %75 = sext i8 %74 to i32
624624 %76 = bitcast i8* bitcast (i32* @si to i8*) to i32*
625 %77 = cmpxchg i32* %76, i32 %73, i32 %75 monotonic
625 %77 = cmpxchg i32* %76, i32 %73, i32 %75 monotonic monotonic
626626 %78 = icmp eq i32 %77, %73
627627 %79 = zext i1 %78 to i8
628628 %80 = zext i8 %79 to i32
632632 %83 = load i8* @sc, align 1
633633 %84 = sext i8 %83 to i32
634634 %85 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
635 %86 = cmpxchg i32* %85, i32 %82, i32 %84 monotonic
635 %86 = cmpxchg i32* %85, i32 %82, i32 %84 monotonic monotonic
636636 %87 = icmp eq i32 %86, %82
637637 %88 = zext i1 %87 to i8
638638 %89 = zext i8 %88 to i32
642642 %92 = load i8* @sc, align 1
643643 %93 = sext i8 %92 to i64
644644 %94 = bitcast i8* bitcast (i64* @sl to i8*) to i64*
645 %95 = cmpxchg i64* %94, i64 %91, i64 %93 monotonic
645 %95 = cmpxchg i64* %94, i64 %91, i64 %93 monotonic monotonic
646646 %96 = icmp eq i64 %95, %91
647647 %97 = zext i1 %96 to i8
648648 %98 = zext i8 %97 to i32
652652 %101 = load i8* @sc, align 1
653653 %102 = sext i8 %101 to i64
654654 %103 = bitcast i8* bitcast (i64* @ul to i8*) to i64*
655 %104 = cmpxchg i64* %103, i64 %100, i64 %102 monotonic
655 %104 = cmpxchg i64* %103, i64 %100, i64 %102 monotonic monotonic
656656 %105 = icmp eq i64 %104, %100
657657 %106 = zext i1 %105 to i8
658658 %107 = zext i8 %106 to i32
1010 define i32 @exchange_and_cmp(i32* %mem) nounwind {
1111 ; CHECK-LABEL: exchange_and_cmp:
1212 ; CHECK: lwarx
13 %tmp = cmpxchg i32* %mem, i32 0, i32 1 monotonic
13 %tmp = cmpxchg i32* %mem, i32 0, i32 1 monotonic monotonic
1414 ; CHECK: stwcx.
1515 ; CHECK: stwcx.
1616 ret i32 %tmp
1010 define i64 @exchange_and_cmp(i64* %mem) nounwind {
1111 ; CHECK-LABEL: exchange_and_cmp:
1212 ; CHECK: ldarx
13 %tmp = cmpxchg i64* %mem, i64 0, i64 1 monotonic
13 %tmp = cmpxchg i64* %mem, i64 0, i64 1 monotonic monotonic
1414 ; CHECK: stdcx.
1515 ; CHECK: stdcx.
1616 ret i64 %tmp
3737
3838 define i32 @test_cmpxchg_i32(i32 %a, i32* %ptr) {
3939 entry:
40 %b = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic
40 %b = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic
4141 ret i32 %b
4242 }
4343
4747
4848 define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) {
4949 entry:
50 %b = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic
50 %b = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
5151 ret i64 %b
5252 }
5353
3131 ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]]
3232 ; CHECK-SHIFT: rll
3333 ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -8([[NEGSHIFT]])
34 %res = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst
34 %res = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst
3535 ret i8 %res
3636 }
3737
4949 ; CHECK-SHIFT: risbg
5050 ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0
5151 ; CHECK-SHIFT: br %r14
52 %res = cmpxchg i8 *%src, i8 42, i8 88 seq_cst
52 %res = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
5353 ret i8 %res
5454 }
3131 ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]]
3232 ; CHECK-SHIFT: rll
3333 ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -16([[NEGSHIFT]])
34 %res = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst
34 %res = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst
3535 ret i16 %res
3636 }
3737
4949 ; CHECK-SHIFT: risbg
5050 ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0
5151 ; CHECK-SHIFT: br %r14
52 %res = cmpxchg i16 *%src, i16 42, i16 88 seq_cst
52 %res = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
5353 ret i16 %res
5454 }
66 ; CHECK-LABEL: f1:
77 ; CHECK: cs %r2, %r3, 0(%r4)
88 ; CHECK: br %r14
9 %val = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst
9 %val = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst
1010 ret i32 %val
1111 }
1212
1616 ; CHECK: cs %r2, %r3, 4092(%r4)
1717 ; CHECK: br %r14
1818 %ptr = getelementptr i32 *%src, i64 1023
19 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
19 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
2020 ret i32 %val
2121 }
2222
2626 ; CHECK: csy %r2, %r3, 4096(%r4)
2727 ; CHECK: br %r14
2828 %ptr = getelementptr i32 *%src, i64 1024
29 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
29 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
3030 ret i32 %val
3131 }
3232
3636 ; CHECK: csy %r2, %r3, 524284(%r4)
3737 ; CHECK: br %r14
3838 %ptr = getelementptr i32 *%src, i64 131071
39 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
39 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
4040 ret i32 %val
4141 }
4242
4848 ; CHECK: cs %r2, %r3, 0(%r4)
4949 ; CHECK: br %r14
5050 %ptr = getelementptr i32 *%src, i64 131072
51 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
51 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
5252 ret i32 %val
5353 }
5454
5858 ; CHECK: csy %r2, %r3, -4(%r4)
5959 ; CHECK: br %r14
6060 %ptr = getelementptr i32 *%src, i64 -1
61 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
61 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
6262 ret i32 %val
6363 }
6464
6868 ; CHECK: csy %r2, %r3, -524288(%r4)
6969 ; CHECK: br %r14
7070 %ptr = getelementptr i32 *%src, i64 -131072
71 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
71 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
7272 ret i32 %val
7373 }
7474
8080 ; CHECK: cs %r2, %r3, 0(%r4)
8181 ; CHECK: br %r14
8282 %ptr = getelementptr i32 *%src, i64 -131073
83 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
83 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
8484 ret i32 %val
8585 }
8686
9292 ; CHECK: br %r14
9393 %add1 = add i64 %src, %index
9494 %ptr = inttoptr i64 %add1 to i32 *
95 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
95 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
9696 ret i32 %val
9797 }
9898
105105 %add1 = add i64 %src, %index
106106 %add2 = add i64 %add1, 4096
107107 %ptr = inttoptr i64 %add2 to i32 *
108 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst
108 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
109109 ret i32 %val
110110 }
111111
115115 ; CHECK: lhi %r2, 1001
116116 ; CHECK: cs %r2, %r3, 0(%r4)
117117 ; CHECK: br %r14
118 %val = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst
118 %val = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst seq_cst
119119 ret i32 %val
120120 }
121121
125125 ; CHECK: lhi [[SWAP:%r[0-9]+]], 1002
126126 ; CHECK: cs %r2, [[SWAP]], 0(%r3)
127127 ; CHECK: br %r14
128 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst
128 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst seq_cst
129129 ret i32 %val
130130 }
66 ; CHECK-LABEL: f1:
77 ; CHECK: csg %r2, %r3, 0(%r4)
88 ; CHECK: br %r14
9 %val = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst
9 %val = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst
1010 ret i64 %val
1111 }
1212
1616 ; CHECK: csg %r2, %r3, 524280(%r4)
1717 ; CHECK: br %r14
1818 %ptr = getelementptr i64 *%src, i64 65535
19 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
19 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
2020 ret i64 %val
2121 }
2222
2828 ; CHECK: csg %r2, %r3, 0(%r4)
2929 ; CHECK: br %r14
3030 %ptr = getelementptr i64 *%src, i64 65536
31 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
31 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
3232 ret i64 %val
3333 }
3434
3838 ; CHECK: csg %r2, %r3, -8(%r4)
3939 ; CHECK: br %r14
4040 %ptr = getelementptr i64 *%src, i64 -1
41 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
41 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
4242 ret i64 %val
4343 }
4444
4848 ; CHECK: csg %r2, %r3, -524288(%r4)
4949 ; CHECK: br %r14
5050 %ptr = getelementptr i64 *%src, i64 -65536
51 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
51 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
5252 ret i64 %val
5353 }
5454
6060 ; CHECK: csg %r2, %r3, 0(%r4)
6161 ; CHECK: br %r14
6262 %ptr = getelementptr i64 *%src, i64 -65537
63 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
63 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
6464 ret i64 %val
6565 }
6666
7272 ; CHECK: br %r14
7373 %add1 = add i64 %src, %index
7474 %ptr = inttoptr i64 %add1 to i64 *
75 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst
75 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
7676 ret i64 %val
7777 }
7878
8282 ; CHECK: lghi %r2, 1001
8383 ; CHECK: csg %r2, %r3, 0(%r4)
8484 ; CHECK: br %r14
85 %val = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst
85 %val = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst seq_cst
8686 ret i64 %val
8787 }
8888
9292 ; CHECK: lghi [[SWAP:%r[0-9]+]], 1002
9393 ; CHECK: csg %r2, [[SWAP]], 0(%r3)
9494 ; CHECK: br %r14
95 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst
95 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst seq_cst
9696 ret i64 %val
9797 }
1717 loop:
1818 ; CHECK: lock
1919 ; CHECK-NEXT: cmpxchg8b
20 %r = cmpxchg i64* %ptr, i64 0, i64 1 monotonic
20 %r = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic
2121 %stored1 = icmp eq i64 %r, 0
2222 br i1 %stored1, label %loop, label %continue
2323 continue:
703703 %3 = zext i8 %2 to i32
704704 %4 = trunc i32 %3 to i8
705705 %5 = trunc i32 %1 to i8
706 %6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic
706 %6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic monotonic
707707 store i8 %6, i8* @sc, align 1
708708 %7 = load i8* @sc, align 1
709709 %8 = zext i8 %7 to i32
711711 %10 = zext i8 %9 to i32
712712 %11 = trunc i32 %10 to i8
713713 %12 = trunc i32 %8 to i8
714 %13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic
714 %13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic monotonic
715715 store i8 %13, i8* @uc, align 1
716716 %14 = load i8* @sc, align 1
717717 %15 = sext i8 %14 to i16
721721 %19 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
722722 %20 = trunc i32 %18 to i16
723723 %21 = trunc i32 %16 to i16
724 %22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic
724 %22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic monotonic
725725 store i16 %22, i16* @ss, align 2
726726 %23 = load i8* @sc, align 1
727727 %24 = sext i8 %23 to i16
731731 %28 = bitcast i8* bitcast (i16* @us to i8*) to i16*
732732 %29 = trunc i32 %27 to i16
733733 %30 = trunc i32 %25 to i16
734 %31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic
734 %31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic monotonic
735735 store i16 %31, i16* @us, align 2
736736 %32 = load i8* @sc, align 1
737737 %33 = sext i8 %32 to i32
738738 %34 = load i8* @uc, align 1
739739 %35 = zext i8 %34 to i32
740740 %36 = bitcast i8* bitcast (i32* @si to i8*) to i32*
741 %37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic
741 %37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic monotonic
742742 store i32 %37, i32* @si, align 4
743743 %38 = load i8* @sc, align 1
744744 %39 = sext i8 %38 to i32
745745 %40 = load i8* @uc, align 1
746746 %41 = zext i8 %40 to i32
747747 %42 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
748 %43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic
748 %43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic monotonic
749749 store i32 %43, i32* @ui, align 4
750750 %44 = load i8* @sc, align 1
751751 %45 = sext i8 %44 to i64
752752 %46 = load i8* @uc, align 1
753753 %47 = zext i8 %46 to i64
754754 %48 = bitcast i8* bitcast (i64* @sl to i8*) to i64*
755 %49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic
755 %49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic monotonic
756756 store i64 %49, i64* @sl, align 8
757757 %50 = load i8* @sc, align 1
758758 %51 = sext i8 %50 to i64
759759 %52 = load i8* @uc, align 1
760760 %53 = zext i8 %52 to i64
761761 %54 = bitcast i8* bitcast (i64* @ul to i8*) to i64*
762 %55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic
762 %55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic monotonic
763763 store i64 %55, i64* @ul, align 8
764764 %56 = load i8* @sc, align 1
765765 %57 = sext i8 %56 to i64
766766 %58 = load i8* @uc, align 1
767767 %59 = zext i8 %58 to i64
768768 %60 = bitcast i8* bitcast (i64* @sll to i8*) to i64*
769 %61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic
769 %61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic monotonic
770770 store i64 %61, i64* @sll, align 8
771771 %62 = load i8* @sc, align 1
772772 %63 = sext i8 %62 to i64
773773 %64 = load i8* @uc, align 1
774774 %65 = zext i8 %64 to i64
775775 %66 = bitcast i8* bitcast (i64* @ull to i8*) to i64*
776 %67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic
776 %67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic monotonic
777777 store i64 %67, i64* @ull, align 8
778778 %68 = load i8* @sc, align 1
779779 %69 = zext i8 %68 to i32
781781 %71 = zext i8 %70 to i32
782782 %72 = trunc i32 %71 to i8
783783 %73 = trunc i32 %69 to i8
784 %74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic
784 %74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic monotonic
785785 %75 = icmp eq i8 %74, %72
786786 %76 = zext i1 %75 to i8
787787 %77 = zext i8 %76 to i32
792792 %81 = zext i8 %80 to i32
793793 %82 = trunc i32 %81 to i8
794794 %83 = trunc i32 %79 to i8
795 %84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic
795 %84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic monotonic
796796 %85 = icmp eq i8 %84, %82
797797 %86 = zext i1 %85 to i8
798798 %87 = zext i8 %86 to i32
804804 %92 = zext i8 %91 to i32
805805 %93 = trunc i32 %92 to i8
806806 %94 = trunc i32 %90 to i8
807 %95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic
807 %95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic monotonic
808808 %96 = icmp eq i8 %95, %93
809809 %97 = zext i1 %96 to i8
810810 %98 = zext i8 %97 to i32
816816 %103 = zext i8 %102 to i32
817817 %104 = trunc i32 %103 to i8
818818 %105 = trunc i32 %101 to i8
819 %106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic
819 %106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic monotonic
820820 %107 = icmp eq i8 %106, %104
821821 %108 = zext i1 %107 to i8
822822 %109 = zext i8 %108 to i32
827827 %113 = zext i8 %112 to i32
828828 %114 = trunc i32 %113 to i8
829829 %115 = trunc i32 %111 to i8
830 %116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic
830 %116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic monotonic
831831 %117 = icmp eq i8 %116, %114
832832 %118 = zext i1 %117 to i8
833833 %119 = zext i8 %118 to i32
838838 %123 = zext i8 %122 to i32
839839 %124 = trunc i32 %123 to i8
840840 %125 = trunc i32 %121 to i8
841 %126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic
841 %126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic monotonic
842842 %127 = icmp eq i8 %126, %124
843843 %128 = zext i1 %127 to i8
844844 %129 = zext i8 %128 to i32
849849 %133 = zext i8 %132 to i64
850850 %134 = trunc i64 %133 to i8
851851 %135 = trunc i64 %131 to i8
852 %136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic
852 %136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic monotonic
853853 %137 = icmp eq i8 %136, %134
854854 %138 = zext i1 %137 to i8
855855 %139 = zext i8 %138 to i32
860860 %143 = zext i8 %142 to i64
861861 %144 = trunc i64 %143 to i8
862862 %145 = trunc i64 %141 to i8
863 %146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic
863 %146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic monotonic
864864 %147 = icmp eq i8 %146, %144
865865 %148 = zext i1 %147 to i8
866866 %149 = zext i8 %148 to i32
871871 %153 = zext i8 %152 to i64
872872 %154 = trunc i64 %153 to i8
873873 %155 = trunc i64 %151 to i8
874 %156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic
874 %156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic monotonic
875875 %157 = icmp eq i8 %156, %154
876876 %158 = zext i1 %157 to i8
877877 %159 = zext i8 %158 to i32
882882 %163 = zext i8 %162 to i64
883883 %164 = trunc i64 %163 to i8
884884 %165 = trunc i64 %161 to i8
885 %166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic
885 %166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic monotonic
886886 %167 = icmp eq i8 %166, %164
887887 %168 = zext i1 %167 to i8
888888 %169 = zext i8 %168 to i32
216216 }
217217
218218 define void @atomic_fetch_cmpxchg16() nounwind {
219 %t1 = cmpxchg i16* @sc16, i16 0, i16 1 acquire
219 %t1 = cmpxchg i16* @sc16, i16 0, i16 1 acquire acquire
220220 ; X64: lock
221221 ; X64: cmpxchgw
222222 ; X32: lock
242242 }
243243
244244 define void @atomic_fetch_cmpxchg32() nounwind {
245 %t1 = cmpxchg i32* @sc32, i32 0, i32 1 acquire
245 %t1 = cmpxchg i32* @sc32, i32 0, i32 1 acquire acquire
246246 ; X64: lock
247247 ; X64: cmpxchgl
248248 ; X32: lock
182182 }
183183
184184 define void @atomic_fetch_cmpxchg64() nounwind {
185 %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire
185 %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire acquire
186186 ; X64: lock
187187 ; X64: cmpxchgq
188188 ; X32: lock
183183 }
184184
185185 define void @atomic_fetch_cmpxchg64() nounwind {
186 %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire
186 %t1 = cmpxchg i64* @sc64, i64 0, i64 1 acquire acquire
187187 ; X32: lock
188188 ; X32: cmpxchg8b
189189 ret void
216216 }
217217
218218 define void @atomic_fetch_cmpxchg8() nounwind {
219 %t1 = cmpxchg i8* @sc8, i8 0, i8 1 acquire
219 %t1 = cmpxchg i8* @sc8, i8 0, i8 1 acquire acquire
220220 ; X64: lock
221221 ; X64: cmpxchgb
222222 ; X32: lock
100100 %neg1 = sub i32 0, 10 ; [#uses=1]
101101 ; CHECK: lock
102102 ; CHECK: cmpxchgl
103 %16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic
103 %16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic monotonic
104104 store i32 %16, i32* %old
105105 ; CHECK: lock
106106 ; CHECK: cmpxchgl
107 %17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic
107 %17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic monotonic
108108 store i32 %17, i32* %old
109109 ; CHECK: movl [[R17atomic:.*]], %eax
110110 ; CHECK: movl $1401, %[[R17mask:[a-z]*]]
132132 ; CHECK: lock
133133 ; CHECK: cmpxchgl %{{.*}}, %gs:(%{{.*}})
134134
135 %0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic
135 %0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic monotonic
136136 ret void
137137 }
55 ; CHECK: movl $1, %ebx
66 ; CHECK: lock
77 ; CHECK-NEXT: cmpxchg16b
8 %r = cmpxchg i128* %p, i128 0, i128 1 seq_cst
8 %r = cmpxchg i128* %p, i128 0, i128 1 seq_cst seq_cst
99 ret void
1010 }
1111
44
55 define i32 @main() nounwind {
66 entry:
7 %0 = cmpxchg i64* @val, i64 0, i64 1 monotonic
7 %0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic
88 %1 = tail call i32 (i8*, ...)* @printf(i8* getelementptr ([7 x i8]* @"\01LC", i32 0, i64 0), i64 %0) nounwind
99 ret i32 0
1010 }
11 define void @test(i128* %a) nounwind {
22 entry:
33 ; CHECK: __sync_val_compare_and_swap_16
4 %0 = cmpxchg i128* %a, i128 1, i128 1 seq_cst
4 %0 = cmpxchg i128* %a, i128 1, i128 1 seq_cst seq_cst
55 ; CHECK: __sync_lock_test_and_set_16
66 %1 = atomicrmw xchg i128* %a, i128 1 seq_cst
77 ; CHECK: __sync_fetch_and_add_16
2525
2626 define void @example_cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) nounwind uwtable sanitize_address {
2727 entry:
28 %0 = cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst
28 %0 = cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
2929 ret void
3030 }
3131
3636
3737 define i32 @Cmpxchg(i32* %p, i32 %a, i32 %b) sanitize_memory {
3838 entry:
39 %0 = cmpxchg i32* %p, i32 %a, i32 %b seq_cst
39 %0 = cmpxchg i32* %p, i32 %a, i32 %b seq_cst seq_cst
4040 ret i32 %0
4141 }
4242
4545 ; CHECK: icmp
4646 ; CHECK: br
4747 ; CHECK: @__msan_warning
48 ; CHECK: cmpxchg {{.*}} seq_cst
48 ; CHECK: cmpxchg {{.*}} seq_cst seq_cst
4949 ; CHECK: store i32 0, {{.*}} @__msan_retval_tls
5050 ; CHECK: ret i32
5151
5252
53 ; relaxed cmpxchg: bump up to "release"
53 ; relaxed cmpxchg: bump up to "release monotonic"
5454
5555 define i32 @CmpxchgMonotonic(i32* %p, i32 %a, i32 %b) sanitize_memory {
5656 entry:
57 %0 = cmpxchg i32* %p, i32 %a, i32 %b monotonic
57 %0 = cmpxchg i32* %p, i32 %a, i32 %b monotonic monotonic
5858 ret i32 %0
5959 }
6060
6363 ; CHECK: icmp
6464 ; CHECK: br
6565 ; CHECK: @__msan_warning
66 ; CHECK: cmpxchg {{.*}} release
66 ; CHECK: cmpxchg {{.*}} release monotonic
6767 ; CHECK: store i32 0, {{.*}} @__msan_retval_tls
6868 ; CHECK: ret i32
6969
347347
348348 define void @atomic8_cas_monotonic(i8* %a) nounwind uwtable {
349349 entry:
350 cmpxchg i8* %a, i8 0, i8 1 monotonic
350 cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic
351351 ret void
352352 }
353353 ; CHECK: atomic8_cas_monotonic
355355
356356 define void @atomic8_cas_acquire(i8* %a) nounwind uwtable {
357357 entry:
358 cmpxchg i8* %a, i8 0, i8 1 acquire
358 cmpxchg i8* %a, i8 0, i8 1 acquire acquire
359359 ret void
360360 }
361361 ; CHECK: atomic8_cas_acquire
363363
364364 define void @atomic8_cas_release(i8* %a) nounwind uwtable {
365365 entry:
366 cmpxchg i8* %a, i8 0, i8 1 release
366 cmpxchg i8* %a, i8 0, i8 1 release monotonic
367367 ret void
368368 }
369369 ; CHECK: atomic8_cas_release
371371
372372 define void @atomic8_cas_acq_rel(i8* %a) nounwind uwtable {
373373 entry:
374 cmpxchg i8* %a, i8 0, i8 1 acq_rel
374 cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire
375375 ret void
376376 }
377377 ; CHECK: atomic8_cas_acq_rel
379379
380380 define void @atomic8_cas_seq_cst(i8* %a) nounwind uwtable {
381381 entry:
382 cmpxchg i8* %a, i8 0, i8 1 seq_cst
382 cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst
383383 ret void
384384 }
385385 ; CHECK: atomic8_cas_seq_cst
731731
732732 define void @atomic16_cas_monotonic(i16* %a) nounwind uwtable {
733733 entry:
734 cmpxchg i16* %a, i16 0, i16 1 monotonic
734 cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic
735735 ret void
736736 }
737737 ; CHECK: atomic16_cas_monotonic
739739
740740 define void @atomic16_cas_acquire(i16* %a) nounwind uwtable {
741741 entry:
742 cmpxchg i16* %a, i16 0, i16 1 acquire
742 cmpxchg i16* %a, i16 0, i16 1 acquire acquire
743743 ret void
744744 }
745745 ; CHECK: atomic16_cas_acquire
747747
748748 define void @atomic16_cas_release(i16* %a) nounwind uwtable {
749749 entry:
750 cmpxchg i16* %a, i16 0, i16 1 release
750 cmpxchg i16* %a, i16 0, i16 1 release monotonic
751751 ret void
752752 }
753753 ; CHECK: atomic16_cas_release
755755
756756 define void @atomic16_cas_acq_rel(i16* %a) nounwind uwtable {
757757 entry:
758 cmpxchg i16* %a, i16 0, i16 1 acq_rel
758 cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire
759759 ret void
760760 }
761761 ; CHECK: atomic16_cas_acq_rel
763763
764764 define void @atomic16_cas_seq_cst(i16* %a) nounwind uwtable {
765765 entry:
766 cmpxchg i16* %a, i16 0, i16 1 seq_cst
766 cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst
767767 ret void
768768 }
769769 ; CHECK: atomic16_cas_seq_cst
11151115
11161116 define void @atomic32_cas_monotonic(i32* %a) nounwind uwtable {
11171117 entry:
1118 cmpxchg i32* %a, i32 0, i32 1 monotonic
1118 cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic
11191119 ret void
11201120 }
11211121 ; CHECK: atomic32_cas_monotonic
11231123
11241124 define void @atomic32_cas_acquire(i32* %a) nounwind uwtable {
11251125 entry:
1126 cmpxchg i32* %a, i32 0, i32 1 acquire
1126 cmpxchg i32* %a, i32 0, i32 1 acquire acquire
11271127 ret void
11281128 }
11291129 ; CHECK: atomic32_cas_acquire
11311131
11321132 define void @atomic32_cas_release(i32* %a) nounwind uwtable {
11331133 entry:
1134 cmpxchg i32* %a, i32 0, i32 1 release
1134 cmpxchg i32* %a, i32 0, i32 1 release monotonic
11351135 ret void
11361136 }
11371137 ; CHECK: atomic32_cas_release
11391139
11401140 define void @atomic32_cas_acq_rel(i32* %a) nounwind uwtable {
11411141 entry:
1142 cmpxchg i32* %a, i32 0, i32 1 acq_rel
1142 cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire
11431143 ret void
11441144 }
11451145 ; CHECK: atomic32_cas_acq_rel
11471147
11481148 define void @atomic32_cas_seq_cst(i32* %a) nounwind uwtable {
11491149 entry:
1150 cmpxchg i32* %a, i32 0, i32 1 seq_cst
1150 cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst
11511151 ret void
11521152 }
11531153 ; CHECK: atomic32_cas_seq_cst
14991499
15001500 define void @atomic64_cas_monotonic(i64* %a) nounwind uwtable {
15011501 entry:
1502 cmpxchg i64* %a, i64 0, i64 1 monotonic
1502 cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic
15031503 ret void
15041504 }
15051505 ; CHECK: atomic64_cas_monotonic
15071507
15081508 define void @atomic64_cas_acquire(i64* %a) nounwind uwtable {
15091509 entry:
1510 cmpxchg i64* %a, i64 0, i64 1 acquire
1510 cmpxchg i64* %a, i64 0, i64 1 acquire acquire
15111511 ret void
15121512 }
15131513 ; CHECK: atomic64_cas_acquire
15151515
15161516 define void @atomic64_cas_release(i64* %a) nounwind uwtable {
15171517 entry:
1518 cmpxchg i64* %a, i64 0, i64 1 release
1518 cmpxchg i64* %a, i64 0, i64 1 release monotonic
15191519 ret void
15201520 }
15211521 ; CHECK: atomic64_cas_release
15231523
15241524 define void @atomic64_cas_acq_rel(i64* %a) nounwind uwtable {
15251525 entry:
1526 cmpxchg i64* %a, i64 0, i64 1 acq_rel
1526 cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire
15271527 ret void
15281528 }
15291529 ; CHECK: atomic64_cas_acq_rel
15311531
15321532 define void @atomic64_cas_seq_cst(i64* %a) nounwind uwtable {
15331533 entry:
1534 cmpxchg i64* %a, i64 0, i64 1 seq_cst
1534 cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst
15351535 ret void
15361536 }
15371537 ; CHECK: atomic64_cas_seq_cst
18831883
18841884 define void @atomic128_cas_monotonic(i128* %a) nounwind uwtable {
18851885 entry:
1886 cmpxchg i128* %a, i128 0, i128 1 monotonic
1886 cmpxchg i128* %a, i128 0, i128 1 monotonic monotonic
18871887 ret void
18881888 }
18891889 ; CHECK: atomic128_cas_monotonic
18911891
18921892 define void @atomic128_cas_acquire(i128* %a) nounwind uwtable {
18931893 entry:
1894 cmpxchg i128* %a, i128 0, i128 1 acquire
1894 cmpxchg i128* %a, i128 0, i128 1 acquire acquire
18951895 ret void
18961896 }
18971897 ; CHECK: atomic128_cas_acquire
18991899
19001900 define void @atomic128_cas_release(i128* %a) nounwind uwtable {
19011901 entry:
1902 cmpxchg i128* %a, i128 0, i128 1 release
1902 cmpxchg i128* %a, i128 0, i128 1 release monotonic
19031903 ret void
19041904 }
19051905 ; CHECK: atomic128_cas_release
19071907
19081908 define void @atomic128_cas_acq_rel(i128* %a) nounwind uwtable {
19091909 entry:
1910 cmpxchg i128* %a, i128 0, i128 1 acq_rel
1910 cmpxchg i128* %a, i128 0, i128 1 acq_rel acquire
19111911 ret void
19121912 }
19131913 ; CHECK: atomic128_cas_acq_rel
19151915
19161916 define void @atomic128_cas_seq_cst(i128* %a) nounwind uwtable {
19171917 entry:
1918 cmpxchg i128* %a, i128 0, i128 1 seq_cst
1918 cmpxchg i128* %a, i128 0, i128 1 seq_cst seq_cst
19191919 ret void
19201920 }
19211921 ; CHECK: atomic128_cas_seq_cst
22 define i8 @cmpswap() {
33 ; CHECK-LABEL: @cmpswap(
44 %i = alloca i8
5 %j = cmpxchg i8* %i, i8 0, i8 42 monotonic
5 %j = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic
66 ; CHECK: [[INST:%[a-z0-9]+]] = load
77 ; CHECK-NEXT: icmp
88 ; CHECK-NEXT: select
6464 entry:
6565 br i1 %C, label %T, label %F
6666 T:
67 cmpxchg volatile i32* %P, i32 0, i32 1 seq_cst
67 cmpxchg volatile i32* %P, i32 0, i32 1 seq_cst seq_cst
6868 unreachable
6969 F:
7070 ret void