llvm.org GIT mirror llvm / 490f68f
[AtomicExpandPass]: Add a hook for custom cmpxchg expansion in IR This involves changing the shouldExpandAtomicCmpXchgInIR interface, but I have updated the in-tree backends using this hook (ARM, AArch64, Hexagon) so they will see no functional change. Previously this hook returned bool, but it now returns AtomicExpansionKind. This hook allows targets to select how a given cmpxchg is to be expanded. D48131 uses this to expand part-word cmpxchg to a target-specific intrinsic. See my associated RFC for more info on the motivation for this change <http://lists.llvm.org/pipermail/llvm-dev/2018-June/123993.html>. Differential Revision: https://reviews.llvm.org/D48130 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@342550 91177308-0d34-0410-b5e6-96231b3b80d8 Alex Bradbury 1 year, 9 months ago
8 changed file(s) with 67 addition(s) and 30 deletion(s). Raw diff Collapse all Expand all
15731573 llvm_unreachable("Masked atomicrmw expansion unimplemented on this target");
15741574 }
15751575
1576 /// Perform a masked cmpxchg using a target-specific intrinsic. This
1577 /// represents the core LL/SC loop which will be lowered at a late stage by
1578 /// the backend.
1579 virtual Value *emitMaskedAtomicCmpXchgIntrinsic(
1580 IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
1581 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
1582 llvm_unreachable("Masked cmpxchg expansion unimplemented on this target");
1583 }
1584
15761585 /// Inserts in the IR a target-specific intrinsic specifying a fence.
15771586 /// It is called by AtomicExpandPass before expanding an
15781587 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
16491658 return AtomicExpansionKind::None;
16501659 }
16511660
1652 /// Returns true if the given atomic cmpxchg should be expanded by the
1653 /// IR-level AtomicExpand pass into a load-linked/store-conditional sequence
1654 /// (through emitLoadLinked() and emitStoreConditional()).
1655 virtual bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
1656 return false;
1661 /// Returns how the given atomic cmpxchg should be expanded by the IR-level
1662 /// AtomicExpand pass.
1663 virtual AtomicExpansionKind
1664 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
1665 return AtomicExpansionKind::None;
16571666 }
16581667
16591668 /// Returns how the IR-level AtomicExpand pass should expand the given
9797 AtomicOrdering MemOpOrder,
9898 function_ref &, Value *)> PerformOp,
9999 CreateCmpXchgInstFun CreateCmpXchg);
100 bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
100101
101102 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
102103 bool isIdempotentRMW(AtomicRMWInst *RMWI);
259260 isAcquireOrStronger(RMWI->getOrdering()))) {
260261 FenceOrdering = RMWI->getOrdering();
261262 RMWI->setOrdering(AtomicOrdering::Monotonic);
262 } else if (CASI && !TLI->shouldExpandAtomicCmpXchgInIR(CASI) &&
263 } else if (CASI &&
264 TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
265 TargetLoweringBase::AtomicExpansionKind::None &&
263266 (isReleaseOrStronger(CASI->getSuccessOrdering()) ||
264267 isAcquireOrStronger(CASI->getSuccessOrdering()))) {
265268 // If a compare and swap is lowered to LL/SC, we can do smarter fence
333336 MadeChange = true;
334337 }
335338
336 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
337 unsigned ValueSize = getAtomicOpSize(CASI);
338 if (ValueSize < MinCASSize) {
339 assert(!TLI->shouldExpandAtomicCmpXchgInIR(CASI) &&
340 "MinCmpXchgSizeInBits not yet supported for LL/SC expansions.");
341 expandPartwordCmpXchg(CASI);
342 } else {
343 if (TLI->shouldExpandAtomicCmpXchgInIR(CASI))
344 MadeChange |= expandAtomicCmpXchg(CASI);
345 }
339 MadeChange |= tryExpandAtomicCmpXchg(CASI);
346340 }
347341 }
348342 return MadeChange;
13541348 return NewLoaded;
13551349 }
13561350
1351 bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
1352 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
1353 unsigned ValueSize = getAtomicOpSize(CI);
1354
1355 switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
1356 default:
1357 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");
1358 case TargetLoweringBase::AtomicExpansionKind::None:
1359 if (ValueSize < MinCASSize)
1360 expandPartwordCmpXchg(CI);
1361 return false;
1362 case TargetLoweringBase::AtomicExpansionKind::LLSC: {
1363 assert(ValueSize >= MinCASSize &&
1364 "MinCmpXchgSizeInBits not yet supported for LL/SC expansions.");
1365 return expandAtomicCmpXchg(CI);
1366 }
1367 case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic:
1368 llvm_unreachable(
1369 "MaskedIntrinsic expansion of cmpxhg not yet implemented");
1370 }
1371 }
1372
13571373 // Note: This function is exposed externally by AtomicExpandUtils.h
13581374 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
13591375 CreateCmpXchgInstFun CreateCmpXchg) {
1137611376 return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC;
1137711377 }
1137811378
11379 bool AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
11379 TargetLowering::AtomicExpansionKind
11380 AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
1138011381 AtomicCmpXchgInst *AI) const {
1138111382 // If subtarget has LSE, leave cmpxchg intact for codegen.
11382 if (Subtarget->hasLSE()) return false;
11383 if (Subtarget->hasLSE())
11384 return AtomicExpansionKind::None;
1138311385 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
1138411386 // implement cmpxchg without spilling. If the address being exchanged is also
1138511387 // on the stack and close enough to the spill slot, this can lead to a
1138611388 // situation where the monitor always gets cleared and the atomic operation
1138711389 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
11388 return getTargetMachine().getOptLevel() != 0;
11390 if (getTargetMachine().getOptLevel() == 0)
11391 return AtomicExpansionKind::None;
11392 return AtomicExpansionKind::LLSC;
1138911393 }
1139011394
1139111395 Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
389389 TargetLoweringBase::AtomicExpansionKind
390390 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
391391
392 bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
392 TargetLoweringBase::AtomicExpansionKind
393 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
393394
394395 bool useLoadStackGuardNode() const override;
395396 TargetLoweringBase::LegalizeTypeAction
1456014560 : AtomicExpansionKind::None;
1456114561 }
1456214562
14563 bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
14564 AtomicCmpXchgInst *AI) const {
14563 TargetLowering::AtomicExpansionKind
14564 ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
1456514565 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
1456614566 // implement cmpxchg without spilling. If the address being exchanged is also
1456714567 // on the stack and close enough to the spill slot, this can lead to a
1456814568 // situation where the monitor always gets cleared and the atomic operation
1456914569 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
14570 bool hasAtomicCmpXchg =
14570 bool HasAtomicCmpXchg =
1457114571 !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps();
14572 return getTargetMachine().getOptLevel() != 0 && hasAtomicCmpXchg;
14572 if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg)
14573 return AtomicExpansionKind::LLSC;
14574 return AtomicExpansionKind::None;
1457314575 }
1457414576
1457514577 bool ARMTargetLowering::shouldInsertFencesForAtomic(
537537 bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
538538 TargetLoweringBase::AtomicExpansionKind
539539 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
540 bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
540 TargetLoweringBase::AtomicExpansionKind
541 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
541542
542543 bool useLoadStackGuardNode() const override;
543544
32133213 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64;
32143214 }
32153215
3216 bool HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3217 AtomicCmpXchgInst *AI) const {
3216 TargetLowering::AtomicExpansionKind
3217 HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3218 AtomicCmpXchgInst *AI) const {
32183219 const DataLayout &DL = AI->getModule()->getDataLayout();
32193220 unsigned Size = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
3220 return Size >= 4 && Size <= 8;
3221 }
3221 if (Size >= 4 && Size <= 8)
3222 return AtomicExpansionKind::LLSC;
3223 return AtomicExpansionKind::None;
3224 }
310310 Value *Addr, AtomicOrdering Ord) const override;
311311 AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
312312 bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
313 bool shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
313 AtomicExpansionKind
314 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
314315
315316 AtomicExpansionKind
316317 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override {