llvm.org GIT mirror llvm / 8f2a85e
IR: add "cmpxchg weak" variant to support permitted failure. This commit adds a weak variant of the cmpxchg operation, as described in C++11. A cmpxchg instruction with this modifier is permitted to fail to store, even if the comparison indicated it should. As a result, cmpxchg instructions must return a flag indicating success in addition to their original iN value loaded. Thus, for uniformity *all* cmpxchg instructions now return "{ iN, i1 }". The second flag is 1 when the store succeeded. At the DAG level, a new ATOMIC_CMP_SWAP_WITH_SUCCESS node has been added as the natural representation for the new cmpxchg instructions. It is a strong cmpxchg. By default this gets Expanded to the existing ATOMIC_CMP_SWAP during Legalization, so existing backends should see no change in behaviour. If they wish to deal with the enhanced node instead, they can call setOperationAction on it. Beware: as a node with 2 results, it cannot be selected from TableGen. Currently, no use is made of the extra information provided in this patch. Test updates are almost entirely adapting the input IR to the new scheme. Summary for out of tree users: ------------------------------ + Legacy Bitcode files are upgraded during read. + Legacy assembly IR files will be invalid. + Front-ends must adapt to different type for "cmpxchg". + Backends should be unaffected by default. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@210903 91177308-0d34-0410-b5e6-96231b3b80d8 Tim Northover 5 years ago
61 changed file(s) with 634 addition(s) and 300 deletion(s). Raw diff Collapse all Expand all
109109
110110 ``cmpxchg`` and ``atomicrmw`` are essentially like an atomic load followed by an
111111 atomic store (where the store is conditional for ``cmpxchg``), but no other
112 memory operation can happen on any thread between the load and store. Note that
113 LLVM's cmpxchg does not provide quite as many options as the C++0x version.
112 memory operation can happen on any thread between the load and store.
114113
115114 A ``fence`` provides Acquire and/or Release ordering which is not part of
116115 another operation; it is normally used along with Monotonic memory operations.
429428 on the users of the result, some ``atomicrmw`` operations can be translated into
430429 operations like ``LOCK AND``, but that does not work in general.
431430
432 On ARM, MIPS, and many other RISC architectures, Acquire, Release, and
433 SequentiallyConsistent semantics require barrier instructions for every such
431 On ARM (before v8), MIPS, and many other RISC architectures, Acquire, Release,
432 and SequentiallyConsistent semantics require barrier instructions for every such
434433 operation. Loads and stores generate normal instructions. ``cmpxchg`` and
435434 ``atomicrmw`` can be represented using a loop with LL/SC-style instructions
436435 which take some sort of exclusive lock on a cache line (``LDREX`` and ``STREX``
437 on ARM, etc.). At the moment, the IR does not provide any way to represent a
438 weak ``cmpxchg`` which would not require a loop.
436 on ARM, etc.).
50585058
50595059 ::
50605060
5061 cmpxchg [volatile] * , , [singlethread] ; yields {ty}
5061 cmpxchg [weak] [volatile] * , , [singlethread] ; yields { , i1 }
50625062
50635063 Overview:
50645064 """""""""
50655065
50665066 The '``cmpxchg``' instruction is used to atomically modify memory. It
50675067 loads a value in memory and compares it to a given value. If they are
5068 equal, it stores a new value into the memory.
5068 equal, it tries to store a new value into the memory.
50695069
50705070 Arguments:
50715071 """"""""""
50985098 Semantics:
50995099 """"""""""
51005100
5101 The contents of memory at the location specified by the '````'
5102 operand is read and compared to '````'; if the read value is the
5103 equal, '````' is written. The original value at the location is
5104 returned.
5101 The contents of memory at the location specified by the '````' operand
5102 is read and compared to '````'; if the read value is the equal, the
5103 '````' is written. The original value at the location is returned, together
5104 with a flag indicating success (true) or failure (false).
5105
5106 If the cmpxchg operation is marked as ``weak`` then a spurious failure is
5107 permitted: the operation may not write ```` even if the comparison
5108 matched.
5109
5110 If the cmpxchg operation is strong (the default), the i1 value is 1 if and only
5111 if the value loaded equals ``cmp``.
51055112
51065113 A successful ``cmpxchg`` is a read-modify-write instruction for the purpose of
51075114 identifying release sequences. A failed ``cmpxchg`` is equivalent to an atomic
51135120 .. code-block:: llvm
51145121
51155122 entry:
5116 %orig = atomic load i32* %ptr unordered ; yields {i32}
5123 %orig = atomic load i32* %ptr unordered ; yields i32
51175124 br label %loop
51185125
51195126 loop:
51205127 %cmp = phi i32 [ %orig, %entry ], [%old, %loop]
51215128 %squared = mul i32 %cmp, %cmp
5122 %old = cmpxchg i32* %ptr, i32 %cmp, i32 %squared acq_rel monotonic ; yields {i32}
5123 %success = icmp eq i32 %cmp, %old
5129 %val_success = cmpxchg i32* %ptr, i32 %cmp, i32 %squared acq_rel monotonic ; yields { i32, i1 }
5130 %value_loaded = extractvalue { i32, i1 } %val_success, 0
5131 %success = extractvalue { i32, i1 } %val_success, 1
51245132 br i1 %success, label %done, label %loop
51255133
51265134 done:
617617 /// swapLo, swapHi)
618618 /// This corresponds to the cmpxchg instruction.
619619 ATOMIC_CMP_SWAP,
620
621 /// Val, Success, OUTCHAIN
622 /// = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap)
623 /// N.b. this is still a strong cmpxchg operation, so
624 /// Success == "Val == cmp".
625 ATOMIC_CMP_SWAP_WITH_SUCCESS,
620626
621627 /// Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
622628 /// Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
699699 SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
700700 SDValue SV, unsigned Align);
701701
702 /// getAtomic - Gets a node for an atomic op, produces result and chain and
703 /// takes 3 operands
704 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
705 SDValue Ptr, SDValue Cmp, SDValue Swp,
706 MachinePointerInfo PtrInfo, unsigned Alignment,
707 AtomicOrdering SuccessOrdering,
708 AtomicOrdering FailureOrdering,
709 SynchronizationScope SynchScope);
710 SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
711 SDValue Ptr, SDValue Cmp, SDValue Swp,
712 MachineMemOperand *MMO,
713 AtomicOrdering SuccessOrdering,
714 AtomicOrdering FailureOrdering,
715 SynchronizationScope SynchScope);
702 /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two
703 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces a the value loaded and a
704 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
705 /// a success flag (initially i1), and a chain.
706 SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs,
707 SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
708 MachinePointerInfo PtrInfo, unsigned Alignment,
709 AtomicOrdering SuccessOrdering,
710 AtomicOrdering FailureOrdering,
711 SynchronizationScope SynchScope);
712 SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs,
713 SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
714 MachineMemOperand *MMO,
715 AtomicOrdering SuccessOrdering,
716 AtomicOrdering FailureOrdering,
717 SynchronizationScope SynchScope);
716718
717719 /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
718720 /// and chain and takes 2 operands.
11251125 N->getOpcode() == ISD::STORE ||
11261126 N->getOpcode() == ISD::PREFETCH ||
11271127 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1128 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
11281129 N->getOpcode() == ISD::ATOMIC_SWAP ||
11291130 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
11301131 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
12331234
12341235 bool isCompareAndSwap() const {
12351236 unsigned Op = getOpcode();
1236 return Op == ISD::ATOMIC_CMP_SWAP;
1237 return Op == ISD::ATOMIC_CMP_SWAP || Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
12371238 }
12381239
12391240 // Methods to support isa and dyn_cast
12401241 static bool classof(const SDNode *N) {
12411242 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1243 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
12421244 N->getOpcode() == ISD::ATOMIC_SWAP ||
12431245 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
12441246 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
497497 void setVolatile(bool V) {
498498 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
499499 (unsigned)V);
500 }
501
502 /// Return true if this cmpxchg may spuriously fail.
503 bool isWeak() const {
504 return getSubclassDataFromInstruction() & 0x100;
505 }
506
507 void setWeak(bool IsWeak) {
508 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x100) |
509 (IsWeak << 8));
500510 }
501511
502512 /// Transparently provide more efficient getOperand methods.
489489 KEYWORD(available_externally);
490490 KEYWORD(linkonce);
491491 KEYWORD(linkonce_odr);
492 KEYWORD(weak);
492 KEYWORD(weak); // Use as a linkage, and a modifier for "cmpxchg".
493493 KEYWORD(weak_odr);
494494 KEYWORD(appending);
495495 KEYWORD(dllimport);
42554255 }
42564256
42574257 /// ParseCmpXchg
4258 /// ::= 'cmpxchg' 'volatile'? TypeAndValue ',' TypeAndValue ',' TypeAndValue
4259 /// 'singlethread'? AtomicOrdering AtomicOrdering
4258 /// ::= 'cmpxchg' 'weak'? 'volatile'? TypeAndValue ',' TypeAndValue ','
4259 /// TypeAndValue 'singlethread'? AtomicOrdering AtomicOrdering
42604260 int LLParser::ParseCmpXchg(Instruction *&Inst, PerFunctionState &PFS) {
42614261 Value *Ptr, *Cmp, *New; LocTy PtrLoc, CmpLoc, NewLoc;
42624262 bool AteExtraComma = false;
42644264 AtomicOrdering FailureOrdering = NotAtomic;
42654265 SynchronizationScope Scope = CrossThread;
42664266 bool isVolatile = false;
4267 bool isWeak = false;
4268
4269 if (EatIfPresent(lltok::kw_weak))
4270 isWeak = true;
42674271
42684272 if (EatIfPresent(lltok::kw_volatile))
42694273 isVolatile = true;
42964300 return Error(NewLoc, "cmpxchg operand must be power-of-two byte-sized"
42974301 " integer");
42984302
4299 AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering,
4300 FailureOrdering, Scope);
4303 AtomicCmpXchgInst *CXI = new AtomicCmpXchgInst(
4304 Ptr, Cmp, New, SuccessOrdering, FailureOrdering, Scope);
43014305 CXI->setVolatile(isVolatile);
4306 CXI->setWeak(isWeak);
43024307 Inst = CXI;
43034308 return AteExtraComma ? InstExtraComma : InstNormal;
43044309 }
4141 kw_linker_private, // NOTE: deprecated, for parser compatibility
4242 kw_linker_private_weak, // NOTE: deprecated, for parser compatibility
4343 kw_linkonce, kw_linkonce_odr,
44 kw_weak, kw_weak_odr, kw_appending,
44 kw_weak, // Used as a linkage, and a modifier for "cmpxchg".
45 kw_weak_odr, kw_appending,
4546 kw_dllimport, kw_dllexport, kw_common, kw_available_externally,
4647 kw_default, kw_hidden, kw_protected,
4748 kw_unnamed_addr,
29222922 }
29232923 case bitc::FUNC_CODE_INST_CMPXCHG: {
29242924 // CMPXCHG:[ptrty, ptr, cmp, new, vol, successordering, synchscope,
2925 // failureordering]
2925 // failureordering?, isweak?]
29262926 unsigned OpNum = 0;
29272927 Value *Ptr, *Cmp, *New;
29282928 if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
29302930 cast(Ptr->getType())->getElementType(), Cmp) ||
29312931 popValue(Record, OpNum, NextValueNo,
29322932 cast(Ptr->getType())->getElementType(), New) ||
2933 (OpNum + 3 != Record.size() && OpNum + 4 != Record.size()))
2933 (Record.size() < OpNum + 3 || Record.size() > OpNum + 5))
29342934 return Error(InvalidRecord);
29352935 AtomicOrdering SuccessOrdering = GetDecodedOrdering(Record[OpNum+1]);
29362936 if (SuccessOrdering == NotAtomic || SuccessOrdering == Unordered)
29472947 I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering,
29482948 SynchScope);
29492949 cast(I)->setVolatile(Record[OpNum]);
2950
2951 if (Record.size() < 8) {
2952 // Before weak cmpxchgs existed, the instruction simply returned the
2953 // value loaded from memory, so bitcode files from that era will be
2954 // expecting the first component of a modern cmpxchg.
2955 CurBB->getInstList().push_back(I);
2956 I = ExtractValueInst::Create(I, 0);
2957 } else {
2958 cast(I)->setWeak(Record[OpNum+4]);
2959 }
2960
29502961 InstructionList.push_back(I);
29512962 break;
29522963 }
14481448 cast(I).getSynchScope()));
14491449 Vals.push_back(GetEncodedOrdering(
14501450 cast(I).getFailureOrdering()));
1451 Vals.push_back(cast(I).isWeak());
14511452 break;
14521453 case Instruction::AtomicRMW:
14531454 Code = bitc::FUNC_CODE_INST_ATOMICRMW;
298298 // Setup the builder so we can create any PHIs we need.
299299 Builder.SetInsertPoint(FailureBB, FailureBB->begin());
300300 BasicBlock *SuccessBB = FailureOrder == Monotonic ? BarrierBB : TryStoreBB;
301 PHINode *Success = nullptr, *Failure = nullptr;
301 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2);
302 Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB);
303 Success->addIncoming(ConstantInt::getFalse(Ctx), LoopBB);
302304
303305 // Look for any users of the cmpxchg that are just comparing the loaded value
304306 // against the desired one, and replace them with the CFG-derived version.
307 SmallVector PrunedInsts;
305308 for (auto User : CI->users()) {
306 ICmpInst *ICmp = dyn_cast(User);
307 if (!ICmp)
309 ExtractValueInst *EV = dyn_cast(User);
310 if (!EV)
308311 continue;
309312
310 // Because we know ICmp uses CI, we only need one operand to be the old
311 // value.
312 if (ICmp->getOperand(0) != CI->getCompareOperand() &&
313 ICmp->getOperand(1) != CI->getCompareOperand())
314 continue;
315
316 if (ICmp->getPredicate() == CmpInst::ICMP_EQ) {
317 if (!Success) {
318 Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2);
319 Success->addIncoming(ConstantInt::getTrue(Ctx), SuccessBB);
320 Success->addIncoming(ConstantInt::getFalse(Ctx), LoopBB);
321 }
322 ICmp->replaceAllUsesWith(Success);
323 } else if (ICmp->getPredicate() == CmpInst::ICMP_NE) {
324 if (!Failure) {
325 Failure = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2);
326 Failure->addIncoming(ConstantInt::getFalse(Ctx), SuccessBB);
327 Failure->addIncoming(ConstantInt::getTrue(Ctx), LoopBB);
328 }
329 ICmp->replaceAllUsesWith(Failure);
330 }
313 assert(EV->getNumIndices() == 1 && EV->getIndices()[0] <= 1 &&
314 "weird extraction from { iN, i1 }");
315
316 if (EV->getIndices()[0] == 0)
317 EV->replaceAllUsesWith(Loaded);
318 else
319 EV->replaceAllUsesWith(Success);
320
321 PrunedInsts.push_back(EV);
331322 }
332323
333 CI->replaceAllUsesWith(Loaded);
324 // We can remove the instructions now we're no longer iterating through them.
325 for (auto EV : PrunedInsts)
326 EV->eraseFromParent();
327
328 if (!CI->use_empty()) {
329 // Some use of the full struct return that we don't understand has happened,
330 // so we've got to reconstruct it properly.
331 Value *Res;
332 Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0);
333 Res = Builder.CreateInsertValue(Res, Success, 1);
334
335 CI->replaceAllUsesWith(Res);
336 }
337
334338 CI->eraseFromParent();
335
336339 return true;
337340 }
338341
30063006 case ISD::ATOMIC_LOAD: {
30073007 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
30083008 SDValue Zero = DAG.getConstant(0, Node->getValueType(0));
3009 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
3010 cast(Node)->getMemoryVT(),
3011 Node->getOperand(0),
3012 Node->getOperand(1), Zero, Zero,
3013 cast(Node)->getMemOperand(),
3014 cast(Node)->getOrdering(),
3015 cast(Node)->getOrdering(),
3016 cast(Node)->getSynchScope());
3009 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3010 SDValue Swap = DAG.getAtomicCmpSwap(
3011 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
3012 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3013 cast(Node)->getMemOperand(),
3014 cast(Node)->getOrdering(),
3015 cast(Node)->getOrdering(),
3016 cast(Node)->getSynchScope());
30173017 Results.push_back(Swap.getValue(0));
30183018 Results.push_back(Swap.getValue(1));
30193019 break;
30483048 std::pair Tmp = ExpandAtomic(Node);
30493049 Results.push_back(Tmp.first);
30503050 Results.push_back(Tmp.second);
3051 break;
3052 }
3053 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3054 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3055 // splits out the success value as a comparison. Expanding the resulting
3056 // ATOMIC_CMP_SWAP will produce a libcall.
3057 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3058 SDValue Res = DAG.getAtomicCmpSwap(
3059 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
3060 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3061 Node->getOperand(3), cast(Node)->getMemOperand(),
3062 cast(Node)->getSuccessOrdering(),
3063 cast(Node)->getFailureOrdering(),
3064 cast(Node)->getSynchScope());
3065
3066 SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1),
3067 Res, Node->getOperand(2), ISD::SETEQ);
3068
3069 Results.push_back(Res.getValue(0));
3070 Results.push_back(Success);
3071 Results.push_back(Res.getValue(1));
30513072 break;
30523073 }
30533074 case ISD::DYNAMIC_STACKALLOC:
137137 Res = PromoteIntRes_Atomic1(cast(N)); break;
138138
139139 case ISD::ATOMIC_CMP_SWAP:
140 Res = PromoteIntRes_Atomic2(cast(N)); break;
140 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
141 Res = PromoteIntRes_AtomicCmpSwap(cast(N), ResNo);
142 break;
141143 }
142144
143145 // If the result is null then the sub-method took care of registering it.
191193 return Res;
192194 }
193195
194 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
196 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
197 unsigned ResNo) {
198 if (ResNo == 1) {
199 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
200 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
201 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
202
203 // Only use the result of getSetCCResultType if it is legal,
204 // otherwise just use the promoted result type (NVT).
205 if (!TLI.isTypeLegal(SVT))
206 SVT = NVT;
207
208 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
209 SDValue Res = DAG.getAtomicCmpSwap(
210 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
211 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
212 N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(),
213 N->getSynchScope());
214 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
215 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
216 return Res.getValue(1);
217 }
218
195219 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
196220 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
197 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
198 N->getChain(), N->getBasePtr(), Op2, Op3,
199 N->getMemOperand(), N->getSuccessOrdering(),
200 N->getFailureOrdering(), N->getSynchScope());
221 SDVTList VTs =
222 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
223 SDValue Res = DAG.getAtomicCmpSwap(
224 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
225 N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(),
226 N->getFailureOrdering(), N->getSynchScope());
201227 // Legalized the chain result - switch anything that used the old chain to
202228 // use the new one.
203 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
229 unsigned ChainOp = N->getNumValues() - 1;
230 ReplaceValueWith(SDValue(N, ChainOp), Res.getValue(ChainOp));
204231 return Res;
205232 }
206233
11401167 std::pair Tmp = ExpandAtomic(N);
11411168 SplitInteger(Tmp.first, Lo, Hi);
11421169 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1170 break;
1171 }
1172 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1173 AtomicSDNode *AN = cast(N);
1174 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1175 SDValue Tmp = DAG.getAtomicCmpSwap(
1176 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1177 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1178 AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(),
1179 AN->getSynchScope());
1180
1181 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1182 // success simply by comparing the loaded value against the ingoing
1183 // comparison.
1184 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1185 N->getOperand(2), ISD::SETEQ);
1186
1187 SplitInteger(Tmp, Lo, Hi);
1188 ReplaceValueWith(SDValue(N, 1), Success);
1189 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
11431190 break;
11441191 }
11451192
23872434 SDValue &Lo, SDValue &Hi) {
23882435 SDLoc dl(N);
23892436 EVT VT = cast(N)->getMemoryVT();
2437 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
23902438 SDValue Zero = DAG.getConstant(0, VT);
2391 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
2392 N->getOperand(0),
2393 N->getOperand(1), Zero, Zero,
2394 cast(N)->getMemOperand(),
2395 cast(N)->getOrdering(),
2396 cast(N)->getOrdering(),
2397 cast(N)->getSynchScope());
2439 SDValue Swap = DAG.getAtomicCmpSwap(
2440 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2441 cast(N)->getMemoryVT(), VTs, N->getOperand(0),
2442 N->getOperand(1), Zero, Zero, cast(N)->getMemOperand(),
2443 cast(N)->getOrdering(),
2444 cast(N)->getOrdering(),
2445 cast(N)->getSynchScope());
2446
23982447 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2399 ReplaceValueWith(SDValue(N, 1), Swap.getValue(1));
2448 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
24002449 }
24012450
24022451 //===----------------------------------------------------------------------===//
219219 SDValue PromoteIntRes_AssertZext(SDNode *N);
220220 SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
221221 SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
222 SDValue PromoteIntRes_Atomic2(AtomicSDNode *N);
222 SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
223223 SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
224224 SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
225225 SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
501501 break;
502502 }
503503 case ISD::ATOMIC_CMP_SWAP:
504 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
504505 case ISD::ATOMIC_SWAP:
505506 case ISD::ATOMIC_LOAD_ADD:
506507 case ISD::ATOMIC_LOAD_SUB:
43264327 Ordering, SynchScope);
43274328 }
43284329
4329 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4330 SDValue Chain, SDValue Ptr, SDValue Cmp,
4331 SDValue Swp, MachinePointerInfo PtrInfo,
4332 unsigned Alignment,
4333 AtomicOrdering SuccessOrdering,
4334 AtomicOrdering FailureOrdering,
4335 SynchronizationScope SynchScope) {
4330 SDValue SelectionDAG::getAtomicCmpSwap(
4331 unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
4332 SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
4333 unsigned Alignment, AtomicOrdering SuccessOrdering,
4334 AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
4335 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4336 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4337 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4338
43364339 if (Alignment == 0) // Ensure that codegen never sees alignment 0
43374340 Alignment = getEVTAlignment(MemVT);
43384341
43394342 MachineFunction &MF = getMachineFunction();
43404343
4341 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE.
4342 // For now, atomics are considered to be volatile always.
43434344 // FIXME: Volatile isn't really correct; we should keep track of atomic
43444345 // orderings in the memoperand.
43454346 unsigned Flags = MachineMemOperand::MOVolatile;
4346 if (Opcode != ISD::ATOMIC_STORE)
4347 Flags |= MachineMemOperand::MOLoad;
4348 if (Opcode != ISD::ATOMIC_LOAD)
4349 Flags |= MachineMemOperand::MOStore;
4347 Flags |= MachineMemOperand::MOLoad;
4348 Flags |= MachineMemOperand::MOStore;
43504349
43514350 MachineMemOperand *MMO =
43524351 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
43534352
4354 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
4353 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
4354 SuccessOrdering, FailureOrdering, SynchScope);
4355 }
4356
4357 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
4358 SDVTList VTs, SDValue Chain, SDValue Ptr,
4359 SDValue Cmp, SDValue Swp,
4360 MachineMemOperand *MMO,
4361 AtomicOrdering SuccessOrdering,
4362 AtomicOrdering FailureOrdering,
4363 SynchronizationScope SynchScope) {
4364 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4365 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4366 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4367
4368 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4369 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
43554370 SuccessOrdering, FailureOrdering, SynchScope);
4356 }
4357
4358 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4359 SDValue Chain,
4360 SDValue Ptr, SDValue Cmp,
4361 SDValue Swp, MachineMemOperand *MMO,
4362 AtomicOrdering SuccessOrdering,
4363 AtomicOrdering FailureOrdering,
4364 SynchronizationScope SynchScope) {
4365 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
4366 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4367
4368 EVT VT = Cmp.getValueType();
4369
4370 SDVTList VTs = getVTList(VT, MVT::Other);
4371 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4372 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, SuccessOrdering,
4373 FailureOrdering, SynchScope);
43744371 }
43754372
43764373 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
36283628 InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
36293629 DAG, *TLI);
36303630
3631 SDValue L =
3632 DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
3633 getValue(I.getCompareOperand()).getSimpleValueType(),
3634 InChain,
3635 getValue(I.getPointerOperand()),
3636 getValue(I.getCompareOperand()),
3637 getValue(I.getNewValOperand()),
3638 MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
3639 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
3640 TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder,
3641 Scope);
3642
3643 SDValue OutChain = L.getValue(1);
3631 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3632 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3633 SDValue L = DAG.getAtomicCmpSwap(
3634 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3635 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
3636 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
3637 0 /* Alignment */,
3638 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
3639 TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope);
3640
3641 SDValue OutChain = L.getValue(2);
36443642
36453643 if (TLI->getInsertFencesForAtomic())
36463644 OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
5454 case ISD::PREFETCH: return "Prefetch";
5555 case ISD::ATOMIC_FENCE: return "AtomicFence";
5656 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap";
57 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: return "AtomicCmpSwapWithSuccess";
5758 case ISD::ATOMIC_SWAP: return "AtomicSwap";
5859 case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd";
5960 case ISD::ATOMIC_LOAD_SUB: return "AtomicLoadSub";
729729 setIndexedStoreAction(IM, (MVT::SimpleValueType)VT, Expand);
730730 }
731731
732 // Most backends expect to see the node which just returns the value loaded.
733 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
734 (MVT::SimpleValueType)VT, Expand);
735
732736 // These operations default to expand.
733737 setOperationAction(ISD::FGETSIGN, (MVT::SimpleValueType)VT, Expand);
734738 setOperationAction(ISD::CONCAT_VECTORS, (MVT::SimpleValueType)VT, Expand);
17851785 (isa(I) && cast(I).isAtomic()))
17861786 Out << " atomic";
17871787
1788 if (isa(I) && cast(I).isWeak())
1789 Out << " weak";
1790
17881791 // If this is a volatile operation, print out the volatile marker.
17891792 if ((isa(I) && cast(I).isVolatile()) ||
17901793 (isa(I) && cast(I).isVolatile()) ||
299299 FI->getSynchScope() == cast(I2)->getSynchScope();
300300 if (const AtomicCmpXchgInst *CXI = dyn_cast(I1))
301301 return CXI->isVolatile() == cast(I2)->isVolatile() &&
302 CXI->isWeak() == cast(I2)->isWeak() &&
302303 CXI->getSuccessOrdering() ==
303304 cast(I2)->getSuccessOrdering() &&
304305 CXI->getFailureOrdering() ==
12501250 AtomicOrdering FailureOrdering,
12511251 SynchronizationScope SynchScope,
12521252 Instruction *InsertBefore)
1253 : Instruction(Cmp->getType(), AtomicCmpXchg,
1254 OperandTraits::op_begin(this),
1255 OperandTraits::operands(this),
1256 InsertBefore) {
1253 : Instruction(
1254 StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext()),
1255 nullptr),
1256 AtomicCmpXchg, OperandTraits::op_begin(this),
1257 OperandTraits::operands(this), InsertBefore) {
12571258 Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope);
12581259 }
12591260
12621263 AtomicOrdering FailureOrdering,
12631264 SynchronizationScope SynchScope,
12641265 BasicBlock *InsertAtEnd)
1265 : Instruction(Cmp->getType(), AtomicCmpXchg,
1266 OperandTraits::op_begin(this),
1267 OperandTraits::operands(this),
1268 InsertAtEnd) {
1266 : Instruction(
1267 StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext()),
1268 nullptr),
1269 AtomicCmpXchg, OperandTraits::op_begin(this),
1270 OperandTraits::operands(this), InsertAtEnd) {
12691271 Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SynchScope);
12701272 }
1271
1273
12721274 //===----------------------------------------------------------------------===//
12731275 // AtomicRMWInst Implementation
12741276 //===----------------------------------------------------------------------===//
36033605 getSuccessOrdering(), getFailureOrdering(),
36043606 getSynchScope());
36053607 Result->setVolatile(isVolatile());
3608 Result->setWeak(isWeak());
36063609 return Result;
36073610 }
36083611
15781578 Out << "\");";
15791579 nl(Out) << iName << "->setVolatile("
15801580 << (cxi->isVolatile() ? "true" : "false") << ");";
1581 nl(Out) << iName << "->setWeak("
1582 << (cxi->isWeak() ? "true" : "false") << ");";
15811583 break;
15821584 }
15831585 case Instruction::AtomicRMW: {
1481214812 // (The only way to get a 16-byte load is cmpxchg16b)
1481314813 // FIXME: 16-byte ATOMIC_CMP_SWAP isn't actually hooked up at the moment.
1481414814 SDValue Zero = DAG.getConstant(0, VT);
14815 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
14816 Node->getOperand(0),
14817 Node->getOperand(1), Zero, Zero,
14818 cast(Node)->getMemOperand(),
14819 cast(Node)->getOrdering(),
14820 cast(Node)->getOrdering(),
14821 cast(Node)->getSynchScope());
14815 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
14816 SDValue Swap =
14817 DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP, dl, VT, VTs,
14818 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
14819 cast(Node)->getMemOperand(),
14820 cast(Node)->getOrdering(),
14821 cast(Node)->getOrdering(),
14822 cast(Node)->getSynchScope());
1482214823 Results.push_back(Swap.getValue(0));
1482314824 Results.push_back(Swap.getValue(1));
1482414825 }
846846 if (int Res = cmpNumbers(CXI->isVolatile(),
847847 cast(R)->isVolatile()))
848848 return Res;
849 if (int Res = cmpNumbers(CXI->isWeak(),
850 cast(R)->isWeak()))
851 return Res;
849852 if (int Res = cmpNumbers(CXI->getSuccessOrdering(),
850853 cast(R)->getSuccessOrdering()))
851854 return Res;
531531 IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false),
532532 createOrdering(&IRB, CASI->getSuccessOrdering()),
533533 createOrdering(&IRB, CASI->getFailureOrdering())};
534 CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef(Args));
535 ReplaceInstWithInst(I, C);
534 CallInst *C = IRB.CreateCall(TsanAtomicCAS[Idx], Args);
535 Value *Success = IRB.CreateICmpEQ(C, CASI->getCompareOperand());
536
537 Value *Res = IRB.CreateInsertValue(UndefValue::get(CASI->getType()), C, 0);
538 Res = IRB.CreateInsertValue(Res, Success, 1);
539
540 I->replaceAllUsesWith(Res);
541 I->eraseFromParent();
536542 } else if (FenceInst *FI = dyn_cast(I)) {
537543 Value *Args[] = {createOrdering(&IRB, FI->getOrdering())};
538544 Function *F = FI->getSynchScope() == SingleThread ?
3131 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
3232 Builder.CreateStore(Res, Ptr);
3333
34 CXI->replaceAllUsesWith(Orig);
34 Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
35 Res = Builder.CreateInsertValue(Res, Equal, 1);
36
37 CXI->replaceAllUsesWith(Res);
3538 CXI->eraseFromParent();
3639 return true;
3740 }
1515 cmpxchg volatile i32* %x, i32 0, i32 1 acq_rel acquire
1616 ; CHECK: cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic
1717 cmpxchg i32* %x, i32 42, i32 0 acq_rel monotonic
18 ; CHECK: cmpxchg weak i32* %x, i32 13, i32 0 seq_cst monotonic
19 cmpxchg weak i32* %x, i32 13, i32 0 seq_cst monotonic
1820 ; CHECK: atomicrmw add i32* %x, i32 10 seq_cst
1921 atomicrmw add i32* %x, i32 10 seq_cst
2022 ; CHECK: atomicrmw volatile xchg i32* %x, i32 10 monotonic
0 ; RUN: llvm-as %s -o - | llvm-dis | FileCheck %s
1
2 define void @test_cmpxchg(i32* %addr, i32 %desired, i32 %new) {
3 cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
4 ; CHECK: cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
5
6 cmpxchg volatile i32* %addr, i32 %desired, i32 %new seq_cst monotonic
7 ; CHECK: cmpxchg volatile i32* %addr, i32 %desired, i32 %new seq_cst monotonic
8
9 cmpxchg weak i32* %addr, i32 %desired, i32 %new acq_rel acquire
10 ; CHECK: cmpxchg weak i32* %addr, i32 %desired, i32 %new acq_rel acquire
11
12 cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread release monotonic
13 ; CHECK: cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread release monotonic
14
15 ret void
16 }
222222 entry:
223223 ;cmpxchg [volatile] * , , [singlethread]
224224
225 ; CHECK: %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
225 ; CHECK: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
226 ; CHECK-NEXT: %res1 = extractvalue { i32, i1 } [[TMP]], 0
226227 %res1 = cmpxchg i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
227228
228 ; CHECK-NEXT: %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
229 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
230 ; CHECK-NEXT: %res2 = extractvalue { i32, i1 } [[TMP]], 0
229231 %res2 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new monotonic monotonic
230232
231 ; CHECK-NEXT: %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
233 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
234 ; CHECK-NEXT: %res3 = extractvalue { i32, i1 } [[TMP]], 0
232235 %res3 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
233236
234 ; CHECK-NEXT: %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
237 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
238 ; CHECK-NEXT: %res4 = extractvalue { i32, i1 } [[TMP]], 0
235239 %res4 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread monotonic monotonic
236240
237241
238 ; CHECK-NEXT: %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
242 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
243 ; CHECK-NEXT: %res5 = extractvalue { i32, i1 } [[TMP]], 0
239244 %res5 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acquire acquire
240245
241 ; CHECK-NEXT: %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
246 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
247 ; CHECK-NEXT: %res6 = extractvalue { i32, i1 } [[TMP]], 0
242248 %res6 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acquire acquire
243249
244 ; CHECK-NEXT: %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
250 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
251 ; CHECK-NEXT: %res7 = extractvalue { i32, i1 } [[TMP]], 0
245252 %res7 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
246253
247 ; CHECK-NEXT: %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
254 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
255 ; CHECK-NEXT: %res8 = extractvalue { i32, i1 } [[TMP]], 0
248256 %res8 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acquire acquire
249257
250258
251 ; CHECK-NEXT: %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
259 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
260 ; CHECK-NEXT: %res9 = extractvalue { i32, i1 } [[TMP]], 0
252261 %res9 = cmpxchg i32* %ptr, i32 %cmp, i32 %new release monotonic
253262
254 ; CHECK-NEXT: %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
263 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
264 ; CHECK-NEXT: %res10 = extractvalue { i32, i1 } [[TMP]], 0
255265 %res10 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new release monotonic
256266
257 ; CHECK-NEXT: %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
267 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
268 ; CHECK-NEXT: %res11 = extractvalue { i32, i1 } [[TMP]], 0
258269 %res11 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
259270
260 ; CHECK-NEXT: %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
271 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
272 ; CHECK-NEXT: %res12 = extractvalue { i32, i1 } [[TMP]], 0
261273 %res12 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread release monotonic
262274
263275
264 ; CHECK-NEXT: %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
276 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
277 ; CHECK-NEXT: %res13 = extractvalue { i32, i1 } [[TMP]], 0
265278 %res13 = cmpxchg i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
266279
267 ; CHECK-NEXT: %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
280 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
281 ; CHECK-NEXT: %res14 = extractvalue { i32, i1 } [[TMP]], 0
268282 %res14 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new acq_rel acquire
269283
270 ; CHECK-NEXT: %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
284 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
285 ; CHECK-NEXT: %res15 = extractvalue { i32, i1 } [[TMP]], 0
271286 %res15 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
272287
273 ; CHECK-NEXT: %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
288 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
289 ; CHECK-NEXT: %res16 = extractvalue { i32, i1 } [[TMP]], 0
274290 %res16 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread acq_rel acquire
275291
276292
277 ; CHECK-NEXT: %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
293 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
294 ; CHECK-NEXT: %res17 = extractvalue { i32, i1 } [[TMP]], 0
278295 %res17 = cmpxchg i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
279296
280 ; CHECK-NEXT: %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
297 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
298 ; CHECK-NEXT: %res18 = extractvalue { i32, i1 } [[TMP]], 0
281299 %res18 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new seq_cst seq_cst
282300
283 ; CHECK-NEXT: %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
301 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
302 ; CHECK-NEXT: %res19 = extractvalue { i32, i1 } [[TMP]], 0
284303 %res19 = cmpxchg i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
285304
286 ; CHECK-NEXT: %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
305 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
306 ; CHECK-NEXT: %res20 = extractvalue { i32, i1 } [[TMP]], 0
287307 %res20 = cmpxchg volatile i32* %ptr, i32 %cmp, i32 %new singlethread seq_cst seq_cst
288308
289309 ret void
0 ; RUN: llvm-dis < %s.bc | FileCheck %s
1
2 ; cmpxchg-upgrade.ll.bc was produced by running a version of llvm-as from just
3 ; before the IR change on this file.
4
5 define i32 @test(i32* %addr, i32 %old, i32 %new) {
6 ; CHECK: [[TMP:%.*]] = cmpxchg i32* %addr, i32 %old, i32 %new seq_cst monotonic
7 ; CHECK: %val = extractvalue { i32, i1 } [[TMP]], 0
8 %val = cmpxchg i32* %addr, i32 %old, i32 %new seq_cst monotonic
9 ret i32 %val
10 }
11
12 define i32 @test(i32* %addr, i32 %old, i32 %new) {
13 ret i1 %val
14 }
1212 ; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]]
1313 ; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]]
1414 ; CHECK: [[DONE]]:
15 %val = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
15 %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
16 %val = extractvalue { i128, i1 } %pair, 0
1617 ret i128 %val
1718 }
1819
99 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[NEWVAL_REG]], [x0]
1010 ; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]]
1111 ; CHECK: [[LABEL2]]:
12 %val = cmpxchg i32* %p, i32 7, i32 4 acquire acquire
12 %pair = cmpxchg i32* %p, i32 7, i32 4 acquire acquire
13 %val = extractvalue { i32, i1 } %pair, 0
1314 ret i32 %val
1415 }
1516
2425 ; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], x[[NEWVAL_REG]], [x0]
2526 ; CHECK: cbnz [[SCRATCH_REG]], [[LABEL]]
2627 ; CHECK: [[LABEL2]]:
27 %val = cmpxchg i64* %p, i64 7, i64 4 monotonic monotonic
28 %pair = cmpxchg i64* %p, i64 7, i64 4 monotonic monotonic
29 %val = extractvalue { i64, i1 } %pair, 0
2830 ret i64 %val
2931 }
3032
877877
878878 define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
879879 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
880 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
880 %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
881 %old = extractvalue { i8, i1 } %pair, 0
882
881883 ; CHECK-NOT: dmb
882884 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
883885 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
898900
899901 define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
900902 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
901 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
903 %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
904 %old = extractvalue { i16, i1 } %pair, 0
905
902906 ; CHECK-NOT: dmb
903907 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
904908 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
919923
920924 define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
921925 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
922 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
926 %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
927 %old = extractvalue { i32, i1 } %pair, 0
928
923929 ; CHECK-NOT: dmb
924930 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
925931 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
940946
941947 define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
942948 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
943 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
949 %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
950 %old = extractvalue { i64, i1 } %pair, 0
951
944952 ; CHECK-NOT: dmb
945953 ; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
946954 ; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1919 ; CHECK: mov w0, wzr
2020 ; CHECK: ret
2121
22 %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
23 %success = icmp eq i32 %loaded, %oldval
22 %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
23 %success = extractvalue { i32, i1 } %pair, 1
2424 %conv = zext i1 %success to i32
2525 ret i32 %conv
2626 }
3737 ; CHECK: cbnz [[STATUS]], [[LOOP]]
3838
3939 ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}}
40 ; CHECK: mov w0, wzr
40 ; FIXME: DAG combine should be able to deal with this.
41 ; CHECK: orr [[TMP:w[0-9]+]], wzr, #0x1
42 ; CHECK: eor w0, [[TMP]], #0x1
4143 ; CHECK: ret
4244
4345 ; CHECK: [[FAILED]]:
4446 ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}}
45 ; CHECK: orr w0, wzr, #0x1
47 ; CHECK: mov [[TMP:w[0-9]+]], wzr
48 ; CHECK: eor w0, [[TMP]], #0x1
4649 ; CHECK: ret
4750
48 %loaded = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic
49 %failure = icmp ne i8 %loaded, %oldValue
51 %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic
52 %success = extractvalue { i8, i1 } %pair, 1
53 %failure = xor i1 %success, 1
5054 ret i1 %failure
5155 }
5256
6872 ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}}
6973 ; CHECK: b _baz
7074
71 %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
72 %success = icmp eq i32 %loaded, %oldval
75 %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
76 %success = extractvalue { i32, i1 } %pair, 1
7377 br i1 %success, label %true, label %false
7478
7579 true:
197197 ; CHECK-THUMB: bne
198198 ; CHECK-THUMB: dmb {{ish$}}
199199
200 %r = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst seq_cst
200 %pair = cmpxchg i64* %ptr, i64 %val1, i64 %val2 seq_cst seq_cst
201 %r = extractvalue { i64, i1 } %pair, 0
201202 ret i64 %r
202203 }
203204
1010 ; T2: ldrexb
1111 ; T2: strexb
1212 %tmp0 = cmpxchg i8* %a, i8 %b, i8 %c monotonic monotonic
13 ret i8 %tmp0
13 %tmp1 = extractvalue { i8, i1 } %tmp0, 0
14 ret i8 %tmp1
1415 }
197197 define i32 @test_cmpxchg_fail_order(i32 *%addr, i32 %desired, i32 %new) {
198198 ; CHECK-LABEL: test_cmpxchg_fail_order:
199199
200 %oldval = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic
200 %pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic
201 %oldval = extractvalue { i32, i1 } %pair, 0
201202 ; CHECK: dmb ish
202203 ; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]:
203204 ; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]]
215216 define i32 @test_cmpxchg_fail_order1(i32 *%addr, i32 %desired, i32 %new) {
216217 ; CHECK-LABEL: test_cmpxchg_fail_order1:
217218
218 %oldval = cmpxchg i32* %addr, i32 %desired, i32 %new acquire acquire
219 %pair = cmpxchg i32* %addr, i32 %desired, i32 %new acquire acquire
220 %oldval = extractvalue { i32, i1 } %pair, 0
219221 ; CHECK-NOT: dmb ish
220222 ; CHECK: [[LOOP_BB:\.?LBB[0-9]+_1]]:
221223 ; CHECK: ldrex [[OLDVAL:r[0-9]+]], [r[[ADDR:[0-9]+]]]
10501050
10511051 define i8 @test_atomic_cmpxchg_i8(i8 zeroext %wanted, i8 zeroext %new) nounwind {
10521052 ; CHECK-LABEL: test_atomic_cmpxchg_i8:
1053 %old = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
1053 %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
1054 %old = extractvalue { i8, i1 } %pair, 0
10541055 ; CHECK-NOT: dmb
10551056 ; CHECK-NOT: mcr
10561057 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
10761077
10771078 define i16 @test_atomic_cmpxchg_i16(i16 zeroext %wanted, i16 zeroext %new) nounwind {
10781079 ; CHECK-LABEL: test_atomic_cmpxchg_i16:
1079 %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
1080 %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
1081 %old = extractvalue { i16, i1 } %pair, 0
10801082 ; CHECK-NOT: dmb
10811083 ; CHECK-NOT: mcr
10821084 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
11021104
11031105 define void @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
11041106 ; CHECK-LABEL: test_atomic_cmpxchg_i32:
1105 %old = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
1107 %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
1108 %old = extractvalue { i32, i1 } %pair, 0
11061109 store i32 %old, i32* @var32
11071110 ; CHECK-NOT: dmb
11081111 ; CHECK-NOT: mcr
11291132
11301133 define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
11311134 ; CHECK-LABEL: test_atomic_cmpxchg_i64:
1132 %old = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
1135 %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
1136 %old = extractvalue { i64, i1 } %pair, 0
11331137 ; CHECK-NOT: dmb
11341138 ; CHECK-NOT: mcr
11351139 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
2424 ; CHECK: dmb ish
2525 ; CHECK: bx lr
2626
27 %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
28 %success = icmp eq i32 %loaded, %oldval
27 %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
28 %success = extractvalue { i32, i1 } %pair, 1
2929 %conv = zext i1 %success to i32
3030 ret i32 %conv
3131 }
3939 ; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
4040 ; CHECK: ldrexb [[LOADED:r[0-9]+]], [r0]
4141 ; CHECK: cmp [[LOADED]], [[OLDBYTE]]
42
43 ; CHECK: itt ne
44 ; CHECK: movne r0, #1
45 ; CHECK: bxne lr
42 ; CHECK: bne [[FAIL:LBB[0-9]+_[0-9]+]]
4643
4744 ; CHECK: strexb [[STATUS:r[0-9]+]], {{r[0-9]+}}, [r0]
4845 ; CHECK: cmp [[STATUS]], #0
4946 ; CHECK: bne [[LOOP]]
5047
48 ; FIXME: this eor is redundant. Need to teach DAG combine that.
5149 ; CHECK-NOT: cmp {{r[0-9]+}}, {{r[0-9]+}}
52 ; CHECK: movs r0, #0
50 ; CHECK: movs [[TMP:r[0-9]+]], #1
51 ; CHECK: eor r0, [[TMP]], #1
5352 ; CHECK: bx lr
5453
55 %loaded = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic
56 %failure = icmp ne i8 %loaded, %oldValue
54 ; CHECK: [[FAIL]]:
55 ; CHECK: movs [[TMP:r[0-9]+]], #0
56 ; CHECK: eor r0, [[TMP]], #1
57 ; CHECK: bx lr
58
59
60 %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic
61 %success = extractvalue { i8, i1 } %pair, 1
62 %failure = xor i1 %success, 1
5763 ret i1 %failure
5864 }
5965
8086 ; CHECK: dmb ish
8187 ; CHECK: b.w _baz
8288
83 %loaded = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
84 %success = icmp eq i32 %loaded, %oldval
89 %pair = cmpxchg i32* %p, i32 %oldval, i32 %newval seq_cst seq_cst
90 %success = extractvalue { i32, i1 } %pair, 1
8591 br i1 %success, label %true, label %false
8692
8793 true:
6464 ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, SequentiallyConsistent, Monotonic, CrossThread
6565 ; CHECK: [[INST]]->setName("inst0");
6666 ; CHECK: [[INST]]->setVolatile(false);
67 ; CHECK: [[INST]]->setWeak(false);
6768
6869 %inst1 = cmpxchg volatile i32* %addr, i32 %desired, i32 %new singlethread acq_rel acquire
6970 ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, AcquireRelease, Acquire, SingleThread
7071 ; CHECK: [[INST]]->setName("inst1");
7172 ; CHECK: [[INST]]->setVolatile(true);
73 ; CHECK: [[INST]]->setWeak(false);
74
75 %inst2 = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst monotonic
76 ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, SequentiallyConsistent, Monotonic, CrossThread
77 ; CHECK: [[INST]]->setName("inst2");
78 ; CHECK: [[INST]]->setVolatile(false);
79 ; CHECK: [[INST]]->setWeak(true);
80
81 %inst3 = cmpxchg weak volatile i32* %addr, i32 %desired, i32 %new singlethread acq_rel acquire
82 ; CHECK: AtomicCmpXchgInst* [[INST:[a-zA-Z0-9_]+]] = new AtomicCmpXchgInst({{.*}}, AcquireRelease, Acquire, SingleThread
83 ; CHECK: [[INST]]->setName("inst3");
84 ; CHECK: [[INST]]->setVolatile(true);
85 ; CHECK: [[INST]]->setWeak(true);
7286
7387 ret void
7488 }
7777 store i32 %newval, i32* %newval.addr, align 4
7878 %tmp = load i32* %newval.addr, align 4
7979 %0 = cmpxchg i32* @x, i32 %oldval, i32 %tmp monotonic monotonic
80 ret i32 %0
80 %1 = extractvalue { i32, i1 } %0, 0
81 ret i32 %1
8182
8283 ; CHECK-EL-LABEL: AtomicCmpSwap32:
8384 ; CHECK-EL: lw $[[R0:[0-9]+]], %got(x)
332333
333334 define signext i8 @AtomicCmpSwap8(i8 signext %oldval, i8 signext %newval) nounwind {
334335 entry:
335 %0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic monotonic
336 %pair0 = cmpxchg i8* @y, i8 %oldval, i8 %newval monotonic monotonic
337 %0 = extractvalue { i8, i1 } %pair0, 0
336338 ret i8 %0
337339
338340 ; CHECK-EL-LABEL: AtomicCmpSwap8:
428430
429431 define i32 @zeroreg() nounwind {
430432 entry:
431 %0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst seq_cst
433 %pair0 = cmpxchg i32* @a, i32 1, i32 0 seq_cst seq_cst
434 %0 = extractvalue { i32, i1 } %pair0, 0
432435 %1 = icmp eq i32 %0, 1
433436 %conv = zext i1 %1 to i32
434437 ret i32 %conv
1919 %add.i = add nsw i32 %0, 2
2020 %1 = load volatile i32* %x, align 4
2121 %call1 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %add.i, i32 %1) nounwind
22 %2 = cmpxchg i32* %x, i32 1, i32 2 seq_cst seq_cst
22 %pair = cmpxchg i32* %x, i32 1, i32 2 seq_cst seq_cst
23 %2 = extractvalue { i32, i1 } %pair, 0
2324 %3 = load volatile i32* %x, align 4
2425 %call2 = call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([8 x i8]* @.str, i32 0, i32 0), i32 %2, i32 %3) nounwind
2526 %4 = atomicrmw xchg i32* %x, i32 1 seq_cst
528528 entry:
529529 %0 = load i8* @uc, align 1
530530 %1 = load i8* @sc, align 1
531 %2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic
531 %pair2 = cmpxchg i8* @sc, i8 %0, i8 %1 monotonic monotonic
532 %2 = extractvalue { i8, i1 } %pair2, 0
532533 store i8 %2, i8* @sc, align 1
533534 %3 = load i8* @uc, align 1
534535 %4 = load i8* @sc, align 1
535 %5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic
536 %pair5 = cmpxchg i8* @uc, i8 %3, i8 %4 monotonic monotonic
537 %5 = extractvalue { i8, i1 } %pair5, 0
536538 store i8 %5, i8* @uc, align 1
537539 %6 = load i8* @uc, align 1
538540 %7 = zext i8 %6 to i16
539541 %8 = load i8* @sc, align 1
540542 %9 = sext i8 %8 to i16
541543 %10 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
542 %11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic
544 %pair11 = cmpxchg i16* %10, i16 %7, i16 %9 monotonic monotonic
545 %11 = extractvalue { i16, i1 } %pair11, 0
543546 store i16 %11, i16* @ss, align 2
544547 %12 = load i8* @uc, align 1
545548 %13 = zext i8 %12 to i16
546549 %14 = load i8* @sc, align 1
547550 %15 = sext i8 %14 to i16
548551 %16 = bitcast i8* bitcast (i16* @us to i8*) to i16*
549 %17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic
552 %pair17 = cmpxchg i16* %16, i16 %13, i16 %15 monotonic monotonic
553 %17 = extractvalue { i16, i1 } %pair17, 0
550554 store i16 %17, i16* @us, align 2
551555 %18 = load i8* @uc, align 1
552556 %19 = zext i8 %18 to i32
553557 %20 = load i8* @sc, align 1
554558 %21 = sext i8 %20 to i32
555559 %22 = bitcast i8* bitcast (i32* @si to i8*) to i32*
556 %23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic
560 %pair23 = cmpxchg i32* %22, i32 %19, i32 %21 monotonic monotonic
561 %23 = extractvalue { i32, i1 } %pair23, 0
557562 store i32 %23, i32* @si, align 4
558563 %24 = load i8* @uc, align 1
559564 %25 = zext i8 %24 to i32
560565 %26 = load i8* @sc, align 1
561566 %27 = sext i8 %26 to i32
562567 %28 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
563 %29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic
568 %pair29 = cmpxchg i32* %28, i32 %25, i32 %27 monotonic monotonic
569 %29 = extractvalue { i32, i1 } %pair29, 0
564570 store i32 %29, i32* @ui, align 4
565571 %30 = load i8* @uc, align 1
566572 %31 = zext i8 %30 to i32
567573 %32 = load i8* @sc, align 1
568574 %33 = sext i8 %32 to i32
569575 %34 = bitcast i8* bitcast (i32* @sl to i8*) to i32*
570 %35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic monotonic
576 %pair35 = cmpxchg i32* %34, i32 %31, i32 %33 monotonic monotonic
577 %35 = extractvalue { i32, i1 } %pair35, 0
571578 store i32 %35, i32* @sl, align 4
572579 %36 = load i8* @uc, align 1
573580 %37 = zext i8 %36 to i32
574581 %38 = load i8* @sc, align 1
575582 %39 = sext i8 %38 to i32
576583 %40 = bitcast i8* bitcast (i32* @ul to i8*) to i32*
577 %41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic monotonic
584 %pair41 = cmpxchg i32* %40, i32 %37, i32 %39 monotonic monotonic
585 %41 = extractvalue { i32, i1 } %pair41, 0
578586 store i32 %41, i32* @ul, align 4
579587 %42 = load i8* @uc, align 1
580588 %43 = load i8* @sc, align 1
581 %44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic
589 %pair44 = cmpxchg i8* @sc, i8 %42, i8 %43 monotonic monotonic
590 %44 = extractvalue { i8, i1 } %pair44, 0
582591 %45 = icmp eq i8 %44, %42
583592 %46 = zext i1 %45 to i32
584593 store i32 %46, i32* @ui, align 4
585594 %47 = load i8* @uc, align 1
586595 %48 = load i8* @sc, align 1
587 %49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic monotonic
596 %pair49 = cmpxchg i8* @uc, i8 %47, i8 %48 monotonic monotonic
597 %49 = extractvalue { i8, i1 } %pair49, 0
588598 %50 = icmp eq i8 %49, %47
589599 %51 = zext i1 %50 to i32
590600 store i32 %51, i32* @ui, align 4
593603 %54 = load i8* @sc, align 1
594604 %55 = sext i8 %54 to i16
595605 %56 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
596 %57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic monotonic
606 %pair57 = cmpxchg i16* %56, i16 %53, i16 %55 monotonic monotonic
607 %57 = extractvalue { i16, i1 } %pair57, 0
597608 %58 = icmp eq i16 %57, %53
598609 %59 = zext i1 %58 to i32
599610 store i32 %59, i32* @ui, align 4
602613 %62 = load i8* @sc, align 1
603614 %63 = sext i8 %62 to i16
604615 %64 = bitcast i8* bitcast (i16* @us to i8*) to i16*
605 %65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic monotonic
616 %pair65 = cmpxchg i16* %64, i16 %61, i16 %63 monotonic monotonic
617 %65 = extractvalue { i16, i1 } %pair65, 0
606618 %66 = icmp eq i16 %65, %61
607619 %67 = zext i1 %66 to i32
608620 store i32 %67, i32* @ui, align 4
611623 %70 = load i8* @sc, align 1
612624 %71 = sext i8 %70 to i32
613625 %72 = bitcast i8* bitcast (i32* @si to i8*) to i32*
614 %73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic monotonic
626 %pair73 = cmpxchg i32* %72, i32 %69, i32 %71 monotonic monotonic
627 %73 = extractvalue { i32, i1 } %pair73, 0
615628 %74 = icmp eq i32 %73, %69
616629 %75 = zext i1 %74 to i32
617630 store i32 %75, i32* @ui, align 4
620633 %78 = load i8* @sc, align 1
621634 %79 = sext i8 %78 to i32
622635 %80 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
623 %81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic monotonic
636 %pair81 = cmpxchg i32* %80, i32 %77, i32 %79 monotonic monotonic
637 %81 = extractvalue { i32, i1 } %pair81, 0
624638 %82 = icmp eq i32 %81, %77
625639 %83 = zext i1 %82 to i32
626640 store i32 %83, i32* @ui, align 4
629643 %86 = load i8* @sc, align 1
630644 %87 = sext i8 %86 to i32
631645 %88 = bitcast i8* bitcast (i32* @sl to i8*) to i32*
632 %89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic monotonic
646 %pair89 = cmpxchg i32* %88, i32 %85, i32 %87 monotonic monotonic
647 %89 = extractvalue { i32, i1 } %pair89, 0
633648 %90 = icmp eq i32 %89, %85
634649 %91 = zext i1 %90 to i32
635650 store i32 %91, i32* @ui, align 4
638653 %94 = load i8* @sc, align 1
639654 %95 = sext i8 %94 to i32
640655 %96 = bitcast i8* bitcast (i32* @ul to i8*) to i32*
641 %97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic monotonic
656 %pair97 = cmpxchg i32* %96, i32 %93, i32 %95 monotonic monotonic
657 %97 = extractvalue { i32, i1 } %pair97, 0
642658 %98 = icmp eq i32 %97, %93
643659 %99 = zext i1 %98 to i32
644660 store i32 %99, i32* @ui, align 4
1010 define i32 @exchange_and_cmp(i32* %mem) nounwind {
1111 ; CHECK-LABEL: exchange_and_cmp:
1212 ; CHECK: lwarx
13 %tmp = cmpxchg i32* %mem, i32 0, i32 1 monotonic monotonic
13 %tmppair = cmpxchg i32* %mem, i32 0, i32 1 monotonic monotonic
14 %tmp = extractvalue { i32, i1 } %tmppair, 0
1415 ; CHECK: stwcx.
1516 ; CHECK: stwcx.
1617 ret i32 %tmp
1010 define i64 @exchange_and_cmp(i64* %mem) nounwind {
1111 ; CHECK-LABEL: exchange_and_cmp:
1212 ; CHECK: ldarx
13 %tmp = cmpxchg i64* %mem, i64 0, i64 1 monotonic monotonic
13 %tmppair = cmpxchg i64* %mem, i64 0, i64 1 monotonic monotonic
14 %tmp = extractvalue { i64, i1 } %tmppair, 0
1415 ; CHECK: stdcx.
1516 ; CHECK: stdcx.
1617 ret i64 %tmp
99 ; SI: S_ENDPGM
1010 define void @lds_atomic_cmpxchg_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr, i32 %swap) nounwind {
1111 %gep = getelementptr i32 addrspace(3)* %ptr, i32 4
12 %result = cmpxchg i32 addrspace(3)* %gep, i32 7, i32 %swap seq_cst monotonic
12 %pair = cmpxchg i32 addrspace(3)* %gep, i32 7, i32 %swap seq_cst monotonic
13 %result = extractvalue { i32, i1 } %pair, 0
1314 store i32 %result, i32 addrspace(1)* %out, align 4
1415 ret void
1516 }
2829 ; SI: S_ENDPGM
2930 define void @lds_atomic_cmpxchg_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr, i64 %swap) nounwind {
3031 %gep = getelementptr i64 addrspace(3)* %ptr, i32 4
31 %result = cmpxchg i64 addrspace(3)* %gep, i64 7, i64 %swap seq_cst monotonic
32 %pair = cmpxchg i64 addrspace(3)* %gep, i64 7, i64 %swap seq_cst monotonic
33 %result = extractvalue { i64, i1 } %pair, 0
3234 store i64 %result, i64 addrspace(1)* %out, align 8
3335 ret void
3436 }
3737
3838 define i32 @test_cmpxchg_i32(i32 %a, i32* %ptr) {
3939 entry:
40 %b = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic
40 %pair = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic monotonic
41 %b = extractvalue { i32, i1 } %pair, 0
4142 ret i32 %b
4243 }
4344
4748
4849 define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) {
4950 entry:
50 %b = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
51 %pair = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic monotonic
52 %b = extractvalue { i64, i1 } %pair, 0
5153 ret i64 %b
5254 }
5355
3131 ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]]
3232 ; CHECK-SHIFT: rll
3333 ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -8([[NEGSHIFT]])
34 %res = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst
34 %pair = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst
35 %res = extractvalue { i8, i1 } %pair, 0
3536 ret i8 %res
3637 }
3738
4950 ; CHECK-SHIFT: risbg
5051 ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0
5152 ; CHECK-SHIFT: br %r14
52 %res = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
53 %pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
54 %res = extractvalue { i8, i1 } %pair, 0
5355 ret i8 %res
5456 }
3131 ; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]]
3232 ; CHECK-SHIFT: rll
3333 ; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -16([[NEGSHIFT]])
34 %res = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst
34 %pair = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst
35 %res = extractvalue { i16, i1 } %pair, 0
3536 ret i16 %res
3637 }
3738
4950 ; CHECK-SHIFT: risbg
5051 ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0
5152 ; CHECK-SHIFT: br %r14
52 %res = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
53 %pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
54 %res = extractvalue { i16, i1 } %pair, 0
5355 ret i16 %res
5456 }
66 ; CHECK-LABEL: f1:
77 ; CHECK: cs %r2, %r3, 0(%r4)
88 ; CHECK: br %r14
9 %val = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst
9 %pair = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst
10 %val = extractvalue { i32, i1 } %pair, 0
1011 ret i32 %val
1112 }
1213
1617 ; CHECK: cs %r2, %r3, 4092(%r4)
1718 ; CHECK: br %r14
1819 %ptr = getelementptr i32 *%src, i64 1023
19 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
20 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
21 %val = extractvalue { i32, i1 } %pair, 0
2022 ret i32 %val
2123 }
2224
2628 ; CHECK: csy %r2, %r3, 4096(%r4)
2729 ; CHECK: br %r14
2830 %ptr = getelementptr i32 *%src, i64 1024
29 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
31 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
32 %val = extractvalue { i32, i1 } %pair, 0
3033 ret i32 %val
3134 }
3235
3639 ; CHECK: csy %r2, %r3, 524284(%r4)
3740 ; CHECK: br %r14
3841 %ptr = getelementptr i32 *%src, i64 131071
39 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
42 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
43 %val = extractvalue { i32, i1 } %pair, 0
4044 ret i32 %val
4145 }
4246
4852 ; CHECK: cs %r2, %r3, 0(%r4)
4953 ; CHECK: br %r14
5054 %ptr = getelementptr i32 *%src, i64 131072
51 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
55 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
56 %val = extractvalue { i32, i1 } %pair, 0
5257 ret i32 %val
5358 }
5459
5863 ; CHECK: csy %r2, %r3, -4(%r4)
5964 ; CHECK: br %r14
6065 %ptr = getelementptr i32 *%src, i64 -1
61 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
66 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
67 %val = extractvalue { i32, i1 } %pair, 0
6268 ret i32 %val
6369 }
6470
6874 ; CHECK: csy %r2, %r3, -524288(%r4)
6975 ; CHECK: br %r14
7076 %ptr = getelementptr i32 *%src, i64 -131072
71 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
77 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
78 %val = extractvalue { i32, i1 } %pair, 0
7279 ret i32 %val
7380 }
7481
8087 ; CHECK: cs %r2, %r3, 0(%r4)
8188 ; CHECK: br %r14
8289 %ptr = getelementptr i32 *%src, i64 -131073
83 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
90 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
91 %val = extractvalue { i32, i1 } %pair, 0
8492 ret i32 %val
8593 }
8694
92100 ; CHECK: br %r14
93101 %add1 = add i64 %src, %index
94102 %ptr = inttoptr i64 %add1 to i32 *
95 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
103 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
104 %val = extractvalue { i32, i1 } %pair, 0
96105 ret i32 %val
97106 }
98107
105114 %add1 = add i64 %src, %index
106115 %add2 = add i64 %add1, 4096
107116 %ptr = inttoptr i64 %add2 to i32 *
108 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
117 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 %swap seq_cst seq_cst
118 %val = extractvalue { i32, i1 } %pair, 0
109119 ret i32 %val
110120 }
111121
115125 ; CHECK: lhi %r2, 1001
116126 ; CHECK: cs %r2, %r3, 0(%r4)
117127 ; CHECK: br %r14
118 %val = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst seq_cst
128 %pair = cmpxchg i32 *%ptr, i32 1001, i32 %swap seq_cst seq_cst
129 %val = extractvalue { i32, i1 } %pair, 0
119130 ret i32 %val
120131 }
121132
125136 ; CHECK: lhi [[SWAP:%r[0-9]+]], 1002
126137 ; CHECK: cs %r2, [[SWAP]], 0(%r3)
127138 ; CHECK: br %r14
128 %val = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst seq_cst
139 %pair = cmpxchg i32 *%ptr, i32 %cmp, i32 1002 seq_cst seq_cst
140 %val = extractvalue { i32, i1 } %pair, 0
129141 ret i32 %val
130142 }
66 ; CHECK-LABEL: f1:
77 ; CHECK: csg %r2, %r3, 0(%r4)
88 ; CHECK: br %r14
9 %val = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst
9 %pairval = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst
10 %val = extractvalue { i64, i1 } %pairval, 0
1011 ret i64 %val
1112 }
1213
1617 ; CHECK: csg %r2, %r3, 524280(%r4)
1718 ; CHECK: br %r14
1819 %ptr = getelementptr i64 *%src, i64 65535
19 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
20 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
21 %val = extractvalue { i64, i1 } %pairval, 0
2022 ret i64 %val
2123 }
2224
2830 ; CHECK: csg %r2, %r3, 0(%r4)
2931 ; CHECK: br %r14
3032 %ptr = getelementptr i64 *%src, i64 65536
31 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
33 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
34 %val = extractvalue { i64, i1 } %pairval, 0
3235 ret i64 %val
3336 }
3437
3841 ; CHECK: csg %r2, %r3, -8(%r4)
3942 ; CHECK: br %r14
4043 %ptr = getelementptr i64 *%src, i64 -1
41 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
44 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
45 %val = extractvalue { i64, i1 } %pairval, 0
4246 ret i64 %val
4347 }
4448
4852 ; CHECK: csg %r2, %r3, -524288(%r4)
4953 ; CHECK: br %r14
5054 %ptr = getelementptr i64 *%src, i64 -65536
51 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
55 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
56 %val = extractvalue { i64, i1 } %pairval, 0
5257 ret i64 %val
5358 }
5459
6065 ; CHECK: csg %r2, %r3, 0(%r4)
6166 ; CHECK: br %r14
6267 %ptr = getelementptr i64 *%src, i64 -65537
63 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
68 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
69 %val = extractvalue { i64, i1 } %pairval, 0
6470 ret i64 %val
6571 }
6672
7278 ; CHECK: br %r14
7379 %add1 = add i64 %src, %index
7480 %ptr = inttoptr i64 %add1 to i64 *
75 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
81 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 %swap seq_cst seq_cst
82 %val = extractvalue { i64, i1 } %pairval, 0
7683 ret i64 %val
7784 }
7885
8289 ; CHECK: lghi %r2, 1001
8390 ; CHECK: csg %r2, %r3, 0(%r4)
8491 ; CHECK: br %r14
85 %val = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst seq_cst
92 %pairval = cmpxchg i64 *%ptr, i64 1001, i64 %swap seq_cst seq_cst
93 %val = extractvalue { i64, i1 } %pairval, 0
8694 ret i64 %val
8795 }
8896
92100 ; CHECK: lghi [[SWAP:%r[0-9]+]], 1002
93101 ; CHECK: csg %r2, [[SWAP]], 0(%r3)
94102 ; CHECK: br %r14
95 %val = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst seq_cst
103 %pairval = cmpxchg i64 *%ptr, i64 %cmp, i64 1002 seq_cst seq_cst
104 %val = extractvalue { i64, i1 } %pairval, 0
96105 ret i64 %val
97106 }
1717 loop:
1818 ; CHECK: lock
1919 ; CHECK-NEXT: cmpxchg8b
20 %r = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic
20 %pair = cmpxchg i64* %ptr, i64 0, i64 1 monotonic monotonic
21 %r = extractvalue { i64, i1 } %pair, 0
2122 %stored1 = icmp eq i64 %r, 0
2223 br i1 %stored1, label %loop, label %continue
2324 continue:
703703 %3 = zext i8 %2 to i32
704704 %4 = trunc i32 %3 to i8
705705 %5 = trunc i32 %1 to i8
706 %6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic monotonic
706 %pair6 = cmpxchg i8* @sc, i8 %4, i8 %5 monotonic monotonic
707 %6 = extractvalue { i8, i1 } %pair6, 0
707708 store i8 %6, i8* @sc, align 1
708709 %7 = load i8* @sc, align 1
709710 %8 = zext i8 %7 to i32
711712 %10 = zext i8 %9 to i32
712713 %11 = trunc i32 %10 to i8
713714 %12 = trunc i32 %8 to i8
714 %13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic monotonic
715 %pair13 = cmpxchg i8* @uc, i8 %11, i8 %12 monotonic monotonic
716 %13 = extractvalue { i8, i1 } %pair13, 0
715717 store i8 %13, i8* @uc, align 1
716718 %14 = load i8* @sc, align 1
717719 %15 = sext i8 %14 to i16
721723 %19 = bitcast i8* bitcast (i16* @ss to i8*) to i16*
722724 %20 = trunc i32 %18 to i16
723725 %21 = trunc i32 %16 to i16
724 %22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic monotonic
726 %pair22 = cmpxchg i16* %19, i16 %20, i16 %21 monotonic monotonic
727 %22 = extractvalue { i16, i1 } %pair22, 0
725728 store i16 %22, i16* @ss, align 2
726729 %23 = load i8* @sc, align 1
727730 %24 = sext i8 %23 to i16
731734 %28 = bitcast i8* bitcast (i16* @us to i8*) to i16*
732735 %29 = trunc i32 %27 to i16
733736 %30 = trunc i32 %25 to i16
734 %31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic monotonic
737 %pair31 = cmpxchg i16* %28, i16 %29, i16 %30 monotonic monotonic
738 %31 = extractvalue { i16, i1 } %pair31, 0
735739 store i16 %31, i16* @us, align 2
736740 %32 = load i8* @sc, align 1
737741 %33 = sext i8 %32 to i32
738742 %34 = load i8* @uc, align 1
739743 %35 = zext i8 %34 to i32
740744 %36 = bitcast i8* bitcast (i32* @si to i8*) to i32*
741 %37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic monotonic
745 %pair37 = cmpxchg i32* %36, i32 %35, i32 %33 monotonic monotonic
746 %37 = extractvalue { i32, i1 } %pair37, 0
742747 store i32 %37, i32* @si, align 4
743748 %38 = load i8* @sc, align 1
744749 %39 = sext i8 %38 to i32
745750 %40 = load i8* @uc, align 1
746751 %41 = zext i8 %40 to i32
747752 %42 = bitcast i8* bitcast (i32* @ui to i8*) to i32*
748 %43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic monotonic
753 %pair43 = cmpxchg i32* %42, i32 %41, i32 %39 monotonic monotonic
754 %43 = extractvalue { i32, i1 } %pair43, 0
749755 store i32 %43, i32* @ui, align 4
750756 %44 = load i8* @sc, align 1
751757 %45 = sext i8 %44 to i64
752758 %46 = load i8* @uc, align 1
753759 %47 = zext i8 %46 to i64
754760 %48 = bitcast i8* bitcast (i64* @sl to i8*) to i64*
755 %49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic monotonic
761 %pair49 = cmpxchg i64* %48, i64 %47, i64 %45 monotonic monotonic
762 %49 = extractvalue { i64, i1 } %pair49, 0
756763 store i64 %49, i64* @sl, align 8
757764 %50 = load i8* @sc, align 1
758765 %51 = sext i8 %50 to i64
759766 %52 = load i8* @uc, align 1
760767 %53 = zext i8 %52 to i64
761768 %54 = bitcast i8* bitcast (i64* @ul to i8*) to i64*
762 %55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic monotonic
769 %pair55 = cmpxchg i64* %54, i64 %53, i64 %51 monotonic monotonic
770 %55 = extractvalue { i64, i1 } %pair55, 0
763771 store i64 %55, i64* @ul, align 8
764772 %56 = load i8* @sc, align 1
765773 %57 = sext i8 %56 to i64
766774 %58 = load i8* @uc, align 1
767775 %59 = zext i8 %58 to i64
768776 %60 = bitcast i8* bitcast (i64* @sll to i8*) to i64*
769 %61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic monotonic
777 %pair61 = cmpxchg i64* %60, i64 %59, i64 %57 monotonic monotonic
778 %61 = extractvalue { i64, i1 } %pair61, 0
770779 store i64 %61, i64* @sll, align 8
771780 %62 = load i8* @sc, align 1
772781 %63 = sext i8 %62 to i64
773782 %64 = load i8* @uc, align 1
774783 %65 = zext i8 %64 to i64
775784 %66 = bitcast i8* bitcast (i64* @ull to i8*) to i64*
776 %67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic monotonic
785 %pair67 = cmpxchg i64* %66, i64 %65, i64 %63 monotonic monotonic
786 %67 = extractvalue { i64, i1 } %pair67, 0
777787 store i64 %67, i64* @ull, align 8
778788 %68 = load i8* @sc, align 1
779789 %69 = zext i8 %68 to i32
781791 %71 = zext i8 %70 to i32
782792 %72 = trunc i32 %71 to i8
783793 %73 = trunc i32 %69 to i8
784 %74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic monotonic
794 %pair74 = cmpxchg i8* @sc, i8 %72, i8 %73 monotonic monotonic
795 %74 = extractvalue { i8, i1 } %pair74, 0
785796 %75 = icmp eq i8 %74, %72
786797 %76 = zext i1 %75 to i8
787798 %77 = zext i8 %76 to i32
792803 %81 = zext i8 %80 to i32
793804 %82 = trunc i32 %81 to i8
794805 %83 = trunc i32 %79 to i8
795 %84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic monotonic
806 %pair84 = cmpxchg i8* @uc, i8 %82, i8 %83 monotonic monotonic
807 %84 = extractvalue { i8, i1 } %pair84, 0
796808 %85 = icmp eq i8 %84, %82
797809 %86 = zext i1 %85 to i8
798810 %87 = zext i8 %86 to i32
804816 %92 = zext i8 %91 to i32
805817 %93 = trunc i32 %92 to i8
806818 %94 = trunc i32 %90 to i8
807 %95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic monotonic
819 %pair95 = cmpxchg i8* bitcast (i16* @ss to i8*), i8 %93, i8 %94 monotonic monotonic
820 %95 = extractvalue { i8, i1 } %pair95, 0
808821 %96 = icmp eq i8 %95, %93
809822 %97 = zext i1 %96 to i8
810823 %98 = zext i8 %97 to i32
816829 %103 = zext i8 %102 to i32
817830 %104 = trunc i32 %103 to i8
818831 %105 = trunc i32 %101 to i8
819 %106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic monotonic
832 %pair106 = cmpxchg i8* bitcast (i16* @us to i8*), i8 %104, i8 %105 monotonic monotonic
833 %106 = extractvalue { i8, i1 } %pair106, 0
820834 %107 = icmp eq i8 %106, %104
821835 %108 = zext i1 %107 to i8
822836 %109 = zext i8 %108 to i32
827841 %113 = zext i8 %112 to i32
828842 %114 = trunc i32 %113 to i8
829843 %115 = trunc i32 %111 to i8
830 %116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic monotonic
844 %pair116 = cmpxchg i8* bitcast (i32* @si to i8*), i8 %114, i8 %115 monotonic monotonic
845 %116 = extractvalue { i8, i1 } %pair116, 0
831846 %117 = icmp eq i8 %116, %114
832847 %118 = zext i1 %117 to i8
833848 %119 = zext i8 %118 to i32
838853 %123 = zext i8 %122 to i32
839854 %124 = trunc i32 %123 to i8
840855 %125 = trunc i32 %121 to i8
841 %126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic monotonic
856 %pair126 = cmpxchg i8* bitcast (i32* @ui to i8*), i8 %124, i8 %125 monotonic monotonic
857 %126 = extractvalue { i8, i1 } %pair126, 0
842858 %127 = icmp eq i8 %126, %124
843859 %128 = zext i1 %127 to i8
844860 %129 = zext i8 %128 to i32
849865 %133 = zext i8 %132 to i64
850866 %134 = trunc i64 %133 to i8
851867 %135 = trunc i64 %131 to i8
852 %136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic monotonic
868 %pair136 = cmpxchg i8* bitcast (i64* @sl to i8*), i8 %134, i8 %135 monotonic monotonic
869 %136 = extractvalue { i8, i1 } %pair136, 0
853870 %137 = icmp eq i8 %136, %134
854871 %138 = zext i1 %137 to i8
855872 %139 = zext i8 %138 to i32
860877 %143 = zext i8 %142 to i64
861878 %144 = trunc i64 %143 to i8
862879 %145 = trunc i64 %141 to i8
863 %146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic monotonic
880 %pair146 = cmpxchg i8* bitcast (i64* @ul to i8*), i8 %144, i8 %145 monotonic monotonic
881 %146 = extractvalue { i8, i1 } %pair146, 0
864882 %147 = icmp eq i8 %146, %144
865883 %148 = zext i1 %147 to i8
866884 %149 = zext i8 %148 to i32
871889 %153 = zext i8 %152 to i64
872890 %154 = trunc i64 %153 to i8
873891 %155 = trunc i64 %151 to i8
874 %156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic monotonic
892 %pair156 = cmpxchg i8* bitcast (i64* @sll to i8*), i8 %154, i8 %155 monotonic monotonic
893 %156 = extractvalue { i8, i1 } %pair156, 0
875894 %157 = icmp eq i8 %156, %154
876895 %158 = zext i1 %157 to i8
877896 %159 = zext i8 %158 to i32
882901 %163 = zext i8 %162 to i64
883902 %164 = trunc i64 %163 to i8
884903 %165 = trunc i64 %161 to i8
885 %166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic monotonic
904 %pair166 = cmpxchg i8* bitcast (i64* @ull to i8*), i8 %164, i8 %165 monotonic monotonic
905 %166 = extractvalue { i8, i1 } %pair166, 0
886906 %167 = icmp eq i8 %166, %164
887907 %168 = zext i1 %167 to i8
888908 %169 = zext i8 %168 to i32
100100 %neg1 = sub i32 0, 10 ; [#uses=1]
101101 ; CHECK: lock
102102 ; CHECK: cmpxchgl
103 %16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic monotonic
103 %pair16 = cmpxchg i32* %val2, i32 %neg1, i32 1 monotonic monotonic
104 %16 = extractvalue { i32, i1 } %pair16, 0
104105 store i32 %16, i32* %old
105106 ; CHECK: lock
106107 ; CHECK: cmpxchgl
107 %17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic monotonic
108 %pair17 = cmpxchg i32* %val2, i32 1976, i32 1 monotonic monotonic
109 %17 = extractvalue { i32, i1 } %pair17, 0
108110 store i32 %17, i32* %old
109111 ; CHECK: movl [[R17atomic:.*]], %eax
110112 ; CHECK: movl $1401, %[[R17mask:[a-z]*]]
132134 ; CHECK: lock
133135 ; CHECK: cmpxchgl %{{.*}}, %gs:(%{{.*}})
134136
135 %0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic monotonic
137 %pair0 = cmpxchg i32 addrspace(256)* %P, i32 0, i32 1 monotonic monotonic
138 %0 = extractvalue { i32, i1 } %pair0, 0
136139 ret void
137140 }
44
55 define i32 @main() nounwind {
66 entry:
7 %0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic
7 %t0 = cmpxchg i64* @val, i64 0, i64 1 monotonic monotonic
8 %0 = extractvalue { i64, i1 } %t0, 0
89 %1 = tail call i32 (i8*, ...)* @printf(i8* getelementptr ([7 x i8]* @"\01LC", i32 0, i64 0), i64 %0) nounwind
910 ret i32 0
1011 }
3636
3737 define i32 @Cmpxchg(i32* %p, i32 %a, i32 %b) sanitize_memory {
3838 entry:
39 %0 = cmpxchg i32* %p, i32 %a, i32 %b seq_cst seq_cst
39 %pair = cmpxchg i32* %p, i32 %a, i32 %b seq_cst seq_cst
40 %0 = extractvalue { i32, i1 } %pair, 0
4041 ret i32 %0
4142 }
4243
4344 ; CHECK: @Cmpxchg
44 ; CHECK: store i32 0,
45 ; CHECK: store { i32, i1 } zeroinitializer,
4546 ; CHECK: icmp
4647 ; CHECK: br
4748 ; CHECK: @__msan_warning
5455
5556 define i32 @CmpxchgMonotonic(i32* %p, i32 %a, i32 %b) sanitize_memory {
5657 entry:
57 %0 = cmpxchg i32* %p, i32 %a, i32 %b monotonic monotonic
58 %pair = cmpxchg i32* %p, i32 %a, i32 %b monotonic monotonic
59 %0 = extractvalue { i32, i1 } %pair, 0
5860 ret i32 %0
5961 }
6062
6163 ; CHECK: @CmpxchgMonotonic
62 ; CHECK: store i32 0,
64 ; CHECK: store { i32, i1 } zeroinitializer,
6365 ; CHECK: icmp
6466 ; CHECK: br
6567 ; CHECK: @__msan_warning
237237 ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]]
238238
239239 ; CHECK: [[BARRIER]]:
240 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ]
240241 ; CHECK: fence seq_cst
241242 ; CHECK: br label %[[DONE:.*]]
242243
243244 ; CHECK: [[DONE]]:
244245 ; CHECK: ret i8 [[OLDVAL]]
245246
246 %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
247 %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
248 %old = extractvalue { i8, i1 } %pairold, 0
247249 ret i8 %old
248250 }
249251
269271 ; CHECK: br label %[[DONE:.*]]
270272
271273 ; CHECK: [[DONE]]:
274 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ]
272275 ; CHECK: ret i16 [[OLDVAL]]
273276
274 %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
277 %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
278 %old = extractvalue { i16, i1 } %pairold, 0
275279 ret i16 %old
276280 }
277281
291295 ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]]
292296
293297 ; CHECK: [[BARRIER]]:
298 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ]
294299 ; CHECK: fence acquire
295300 ; CHECK: br label %[[DONE:.*]]
296301
297302 ; CHECK: [[DONE]]:
298303 ; CHECK: ret i32 [[OLDVAL]]
299304
300 %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
305 %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
306 %old = extractvalue { i32, i1 } %pairold, 0
301307 ret i32 %old
302308 }
303309
332338 ; CHECK: br label %[[DONE:.*]]
333339
334340 ; CHECK: [[DONE]]:
341 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ]
335342 ; CHECK: ret i64 [[OLDVAL]]
336343
337 %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
344 %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
345 %old = extractvalue { i64, i1 } %pairold, 0
338346 ret i64 %old
339347 }
9999 ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]]
100100
101101 ; CHECK: [[BARRIER]]:
102 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ]
102103 ; CHECK-NOT: fence
103104 ; CHECK: br label %[[DONE:.*]]
104105
105106 ; CHECK: [[DONE]]:
106107 ; CHECK: ret i8 [[OLDVAL]]
107108
108 %old = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
109 %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
110 %old = extractvalue { i8, i1 } %pairold, 0
109111 ret i8 %old
110112 }
111113
131133 ; CHECK: br label %[[DONE:.*]]
132134
133135 ; CHECK: [[DONE]]:
136 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[BARRIER]] ], [ false, %[[LOOP]] ]
134137 ; CHECK: ret i16 [[OLDVAL]]
135138
136 %old = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
139 %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
140 %old = extractvalue { i16, i1 } %pairold, 0
137141 ret i16 %old
138142 }
139143
153157 ; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[BARRIER:.*]]
154158
155159 ; CHECK: [[BARRIER]]:
160 ; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[TRY_STORE]] ], [ false, %[[LOOP]] ]
156161 ; CHECK-NOT: fence
157162 ; CHECK: br label %[[DONE:.*]]
158163
159164 ; CHECK: [[DONE]]:
160165 ; CHECK: ret i32 [[OLDVAL]]
161166
162 %old = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
167 %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
168 %old = extractvalue { i32, i1 } %pairold, 0
163169 ret i32 %old
164170 }
165171
196202 ; CHECK: [[DONE]]:
197203 ; CHECK: ret i64 [[OLDVAL]]
198204
199 %old = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
205 %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
206 %old = extractvalue { i64, i1 } %pairold, 0
200207 ret i64 %old
201208 }
22 define i8 @cmpswap() {
33 ; CHECK-LABEL: @cmpswap(
44 %i = alloca i8
5 %j = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic
6 ; CHECK: [[INST:%[a-z0-9]+]] = load
7 ; CHECK-NEXT: icmp
8 ; CHECK-NEXT: select
9 ; CHECK-NEXT: store
5 %pair = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic
6 %j = extractvalue { i8, i1 } %pair, 0
7 ; CHECK: [[OLDVAL:%[a-z0-9]+]] = load i8* [[ADDR:%[a-z0-9]+]]
8 ; CHECK-NEXT: [[SAME:%[a-z0-9]+]] = icmp eq i8 [[OLDVAL]], 0
9 ; CHECK-NEXT: [[TO_STORE:%[a-z0-9]+]] = select i1 [[SAME]], i8 42, i8 [[OLDVAL]]
10 ; CHECK-NEXT: store i8 [[TO_STORE]], i8* [[ADDR]]
11 ; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = insertvalue { i8, i1 } undef, i8 [[OLDVAL]], 0
12 ; CHECK-NEXT: [[RES:%[a-z0-9]+]] = insertvalue { i8, i1 } [[TMP]], i1 [[SAME]], 1
13 ; CHECK-NEXT: [[VAL:%[a-z0-9]+]] = extractvalue { i8, i1 } [[RES]], 0
1014 ret i8 %j
11 ; CHECK: ret i8 [[INST]]
15 ; CHECK: ret i8 [[VAL]]
1216 }
17
1318
1419 define i8 @swap() {
1520 ; CHECK-LABEL: @swap(