llvm.org GIT mirror llvm / 1ad925c
Refactor AtomicExpandPass and add a generic isAtomic() method to Instruction Summary: Split shouldExpandAtomicInIR() into different versions for Stores/Loads/RMWs/CmpXchgs. Makes runOnFunction cleaner (no more redundant checking/casting), and will help moving the X86 backend to this pass. This requires a way of easily detecting which instructions are atomic. I followed the pattern of mayReadFromMemory, mayWriteOrReadMemory, etc.. in making isAtomic() a method of Instruction implemented by a switch on the opcodes. Test Plan: make check Reviewers: jfb Subscribers: mcrosier, llvm-commits Differential Revision: http://reviews.llvm.org/D5035 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@217080 91177308-0d34-0410-b5e6-96231b3b80d8 Robin Morisset 5 years ago
9 changed file(s) with 127 addition(s) and 80 deletion(s). Raw diff Collapse all Expand all
337337 return mayReadFromMemory() || mayWriteToMemory();
338338 }
339339
340 /// isAtomic - Return true if this instruction has an
341 /// AtomicOrdering of unordered or higher.
342 ///
343 bool isAtomic() const;
344
340345 /// mayThrow - Return true if this instruction may throw an exception.
341346 ///
342347 bool mayThrow() const;
240240 (xthread << 6));
241241 }
242242
243 bool isAtomic() const { return getOrdering() != NotAtomic; }
244243 void setAtomic(AtomicOrdering Ordering,
245244 SynchronizationScope SynchScope = CrossThread) {
246245 setOrdering(Ordering);
360359 (xthread << 6));
361360 }
362361
363 bool isAtomic() const { return getOrdering() != NotAtomic; }
364362 void setAtomic(AtomicOrdering Ordering,
365363 SynchronizationScope SynchScope = CrossThread) {
366364 setOrdering(Ordering);
3030 #include "llvm/IR/CallSite.h"
3131 #include "llvm/IR/CallingConv.h"
3232 #include "llvm/IR/InlineAsm.h"
33 #include "llvm/IR/Instructions.h"
3334 #include "llvm/IR/IRBuilder.h"
3435 #include "llvm/MC/MCRegisterInfo.h"
3536 #include "llvm/Target/TargetCallingConv.h"
954955 /// It is called by AtomicExpandPass before expanding an
955956 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
956957 /// RMW and CmpXchg set both IsStore and IsLoad to true.
957 /// Backends with !getInsertFencesForAtomic() should keep a no-op here
958 /// Backends with !getInsertFencesForAtomic() should keep a no-op here.
958959 virtual void emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
959960 bool IsStore, bool IsLoad) const {
960961 assert(!getInsertFencesForAtomic());
964965 /// It is called by AtomicExpandPass after expanding an
965966 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
966967 /// RMW and CmpXchg set both IsStore and IsLoad to true.
967 /// Backends with !getInsertFencesForAtomic() should keep a no-op here
968 /// Backends with !getInsertFencesForAtomic() should keep a no-op here.
968969 virtual void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
969970 bool IsStore, bool IsLoad) const {
970971 assert(!getInsertFencesForAtomic());
971972 }
972973
973 /// Return true if the given (atomic) instruction should be expanded by the
974 /// IR-level AtomicExpand pass into a loop involving
975 /// load-linked/store-conditional pairs. Atomic stores will be expanded in the
976 /// same way as "atomic xchg" operations which ignore their output if needed.
977 virtual bool shouldExpandAtomicInIR(Instruction *Inst) const {
978 return false;
979 }
980
974 /// Returns true if the given (atomic) store should be expanded by the
975 /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input.
976 virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const {
977 return false;
978 }
979
980 /// Returns true if the given (atomic) load should be expanded by the
981 /// IR-level AtomicExpand pass into a load-linked instruction
982 /// (through emitLoadLinked()).
983 virtual bool shouldExpandAtomicLoadInIR(LoadInst *LI) const { return false; }
984
985 /// Returns true if the given AtomicRMW should be expanded by the
986 /// IR-level AtomicExpand pass into a loop using LoadLinked/StoreConditional.
987 virtual bool shouldExpandAtomicRMWInIR(AtomicRMWInst *RMWI) const {
988 return false;
989 }
981990
982991 //===--------------------------------------------------------------------===//
983992 // TargetLowering Configuration Methods - These methods should be invoked by
1414 #include "llvm/CodeGen/Passes.h"
1515 #include "llvm/IR/Function.h"
1616 #include "llvm/IR/IRBuilder.h"
17 #include "llvm/IR/InstIterator.h"
1718 #include "llvm/IR/Instructions.h"
1819 #include "llvm/IR/Intrinsics.h"
1920 #include "llvm/IR/Module.h"
3738 }
3839
3940 bool runOnFunction(Function &F) override;
40 bool expandAtomicInsts(Function &F);
41
41
42 private:
4243 bool expandAtomicLoad(LoadInst *LI);
43 bool expandAtomicStore(StoreInst *LI);
44 bool expandAtomicStore(StoreInst *SI);
4445 bool expandAtomicRMW(AtomicRMWInst *AI);
4546 bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
4647 };
5960 bool AtomicExpand::runOnFunction(Function &F) {
6061 if (!TM || !TM->getSubtargetImpl()->enableAtomicExpand())
6162 return false;
63 auto TargetLowering = TM->getSubtargetImpl()->getTargetLowering();
6264
6365 SmallVector AtomicInsts;
6466
6567 // Changing control-flow while iterating through it is a bad idea, so gather a
6668 // list of all atomic instructions before we start.
67 for (BasicBlock &BB : F)
68 for (Instruction &Inst : BB) {
69 if (isa(&Inst) || isa(&Inst) ||
70 (isa(&Inst) && cast(&Inst)->isAtomic()) ||
71 (isa(&Inst) && cast(&Inst)->isAtomic()))
72 AtomicInsts.push_back(&Inst);
69 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) {
70 if (I->isAtomic())
71 AtomicInsts.push_back(&*I);
72 }
73
74 bool MadeChange = false;
75 for (auto I : AtomicInsts) {
76 auto LI = dyn_cast(I);
77 auto SI = dyn_cast(I);
78 auto RMWI = dyn_cast(I);
79 auto CASI = dyn_cast(I);
80
81 assert((LI || SI || RMWI || CASI || isa(I)) &&
82 "Unknown atomic instruction");
83
84 if (LI && TargetLowering->shouldExpandAtomicLoadInIR(LI)) {
85 MadeChange |= expandAtomicLoad(LI);
86 } else if (SI && TargetLowering->shouldExpandAtomicStoreInIR(SI)) {
87 MadeChange |= expandAtomicStore(SI);
88 } else if (RMWI && TargetLowering->shouldExpandAtomicRMWInIR(RMWI)) {
89 MadeChange |= expandAtomicRMW(RMWI);
90 } else if (CASI) {
91 MadeChange |= expandAtomicCmpXchg(CASI);
7392 }
74
75 bool MadeChange = false;
76 for (Instruction *Inst : AtomicInsts) {
77 if (!TM->getSubtargetImpl()->getTargetLowering()->shouldExpandAtomicInIR(
78 Inst))
79 continue;
80
81 if (AtomicRMWInst *AI = dyn_cast(Inst))
82 MadeChange |= expandAtomicRMW(AI);
83 else if (AtomicCmpXchgInst *CI = dyn_cast(Inst))
84 MadeChange |= expandAtomicCmpXchg(CI);
85 else if (LoadInst *LI = dyn_cast(Inst))
86 MadeChange |= expandAtomicLoad(LI);
87 else if (StoreInst *SI = dyn_cast(Inst))
88 MadeChange |= expandAtomicStore(SI);
89 else
90 llvm_unreachable("Unknown atomic instruction");
91 }
92
93 }
9394 return MadeChange;
9495 }
9596
145146 BasicBlock *BB = AI->getParent();
146147 Function *F = BB->getParent();
147148 LLVMContext &Ctx = F->getContext();
148 // If getInsertFencesForAtomic() return true, then the target does not want to
149 // deal with memory orders, and emitLeading/TrailingFence should take care of
150 // everything. Otherwise, emitLeading/TrailingFence are no-op and we should
151 // preserve the ordering.
149 // If getInsertFencesForAtomic() returns true, then the target does not want
150 // to deal with memory orders, and emitLeading/TrailingFence should take care
151 // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
152 // should preserve the ordering.
152153 AtomicOrdering MemOpOrder =
153154 TLI->getInsertFencesForAtomic() ? Monotonic : Order;
154155
251252 BasicBlock *BB = CI->getParent();
252253 Function *F = BB->getParent();
253254 LLVMContext &Ctx = F->getContext();
254 // If getInsertFencesForAtomic() return true, then the target does not want to
255 // deal with memory orders, and emitLeading/TrailingFence should take care of
256 // everything. Otherwise, emitLeading/TrailingFence are no-op and we should
257 // preserve the ordering.
255 // If getInsertFencesForAtomic() returns true, then the target does not want
256 // to deal with memory orders, and emitLeading/TrailingFence should take care
257 // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
258 // should preserve the ordering.
258259 AtomicOrdering MemOpOrder =
259260 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder;
260261
442442 }
443443 }
444444
445 bool Instruction::isAtomic() const {
446 switch (getOpcode()) {
447 default:
448 return false;
449 case Instruction::AtomicCmpXchg:
450 case Instruction::AtomicRMW:
451 case Instruction::Fence:
452 return true;
453 case Instruction::Load:
454 return cast(this)->getOrdering() != NotAtomic;
455 case Instruction::Store:
456 return cast(this)->getOrdering() != NotAtomic;
457 }
458 }
459
445460 bool Instruction::mayThrow() const {
446461 if (const CallInst *CI = dyn_cast(this))
447462 return !CI->doesNotThrow();
85128512 }
85138513 }
85148514
8515 bool AArch64TargetLowering::shouldExpandAtomicInIR(Instruction *Inst) const {
8516 // Loads and stores less than 128-bits are already atomic; ones above that
8517 // are doomed anyway, so defer to the default libcall and blame the OS when
8518 // things go wrong:
8519 if (StoreInst *SI = dyn_cast(Inst))
8520 return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 128;
8521 else if (LoadInst *LI = dyn_cast(Inst))
8522 return LI->getType()->getPrimitiveSizeInBits() == 128;
8523
8524 // For the real atomic operations, we have ldxr/stxr up to 128 bits.
8525 return Inst->getType()->getPrimitiveSizeInBits() <= 128;
8526 }
8527
85288515 bool AArch64TargetLowering::useLoadStackGuardNode() const {
85298516 return true;
85308517 }
85418528 return TargetLoweringBase::getPreferredVectorAction(VT);
85428529 }
85438530
8531 // Loads and stores less than 128-bits are already atomic; ones above that
8532 // are doomed anyway, so defer to the default libcall and blame the OS when
8533 // things go wrong.
8534 bool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
8535 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
8536 return Size == 128;
8537 }
8538
8539 // Loads and stores less than 128-bits are already atomic; ones above that
8540 // are doomed anyway, so defer to the default libcall and blame the OS when
8541 // things go wrong.
8542 bool AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
8543 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
8544 return Size == 128;
8545 }
8546
8547 // For the real atomic operations, we have ldxr/stxr up to 128 bits,
8548 bool AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
8549 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
8550 return Size <= 128;
8551 }
8552
85448553 Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
85458554 AtomicOrdering Ord) const {
85468555 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
321321 Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
322322 Value *Addr, AtomicOrdering Ord) const override;
323323
324 bool shouldExpandAtomicInIR(Instruction *Inst) const override;
324 bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
325 bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
326 bool shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
325327
326328 bool useLoadStackGuardNode() const override;
327329 TargetLoweringBase::LegalizeTypeAction
1103811038 }
1103911039 }
1104011040
11041 bool ARMTargetLowering::shouldExpandAtomicInIR(Instruction *Inst) const {
11042 // Loads and stores less than 64-bits are already atomic; ones above that
11043 // are doomed anyway, so defer to the default libcall and blame the OS when
11044 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
11045 // anything for those.
11046 bool IsMClass = Subtarget->isMClass();
11047 if (StoreInst *SI = dyn_cast(Inst)) {
11048 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
11049 return Size == 64 && !IsMClass;
11050 } else if (LoadInst *LI = dyn_cast(Inst)) {
11051 return LI->getType()->getPrimitiveSizeInBits() == 64 && !IsMClass;
11052 }
11053
11054 // For the real atomic operations, we have ldrex/strex up to 32 bits,
11055 // and up to 64 bits on the non-M profiles
11056 unsigned AtomicLimit = IsMClass ? 32 : 64;
11057 return Inst->getType()->getPrimitiveSizeInBits() <= AtomicLimit;
11041 // Loads and stores less than 64-bits are already atomic; ones above that
11042 // are doomed anyway, so defer to the default libcall and blame the OS when
11043 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
11044 // anything for those.
11045 bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
11046 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
11047 return (Size == 64) && !Subtarget->isMClass();
11048 }
11049
11050 // Loads and stores less than 64-bits are already atomic; ones above that
11051 // are doomed anyway, so defer to the default libcall and blame the OS when
11052 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
11053 // anything for those.
11054 bool ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
11055 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
11056 return (Size == 64) && !Subtarget->isMClass();
11057 }
11058
11059 // For the real atomic operations, we have ldrex/strex up to 32 bits,
11060 // and up to 64 bits on the non-M profiles
11061 bool ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
11062 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11063 return Size <= (Subtarget->isMClass() ? 32 : 64);
1105811064 }
1105911065
1106011066 // This has so far only been implemented for MachO.
401401 void emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
402402 bool IsStore, bool IsLoad) const override;
403403
404 bool shouldExpandAtomicInIR(Instruction *Inst) const override;
404 bool shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
405 bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
406 bool shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
405407
406408 bool useLoadStackGuardNode() const override;
407409