llvm.org GIT mirror llvm / ceb61bf
Tweak some atomics functions in preparation for larger changes; NFC. - Rename getATOMIC to getSYNC, as llvm will soon be able to emit both '__sync' libcalls and '__atomic' libcalls, and this function is for the '__sync' ones. - getInsertFencesForAtomic() has been replaced with shouldInsertFencesForAtomic(Instruction), so that the decision can be made per-instruction. This functionality will be used soon. - emitLeadingFence/emitTrailingFence are no longer called if shouldInsertFencesForAtomic returns false, and thus don't need to check the condition themselves. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@263665 91177308-0d34-0410-b5e6-96231b3b80d8 James Y Knight 4 years ago
17 changed file(s) with 58 addition(s) and 56 deletion(s). Raw diff Collapse all Expand all
429429
430430 /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
431431 /// UNKNOWN_LIBCALL if there is none.
432 Libcall getATOMIC(unsigned Opc, MVT VT);
432 Libcall getSYNC(unsigned Opc, MVT VT);
433433 }
434434 }
435435
10021002 return PrefLoopAlignment;
10031003 }
10041004
1005 /// Return whether the DAG builder should automatically insert fences and
1006 /// reduce ordering for atomics.
1007 bool getInsertFencesForAtomic() const {
1008 return InsertFencesForAtomic;
1009 }
1010
10111005 /// Return true if the target stores stack protector cookies at a fixed offset
10121006 /// in some non-standard address space, and populates the address space and
10131007 /// offset as appropriate.
10501044 //===--------------------------------------------------------------------===//
10511045 /// \name Helpers for atomic expansion.
10521046 /// @{
1047
1048 /// Whether AtomicExpandPass should automatically insert fences and reduce
1049 /// ordering for this atomic. This should be true for most architectures with
1050 /// weak memory ordering. Defaults to false.
1051 virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
1052 return false;
1053 }
10531054
10541055 /// Perform a load-linked operation on Addr, returning a "Value *" with the
10551056 /// corresponding pointee type. This may entail some non-trivial operations to
10691070
10701071 /// Inserts in the IR a target-specific intrinsic specifying a fence.
10711072 /// It is called by AtomicExpandPass before expanding an
1072 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
1073 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
1074 /// if shouldInsertFencesForAtomic returns true.
10731075 /// RMW and CmpXchg set both IsStore and IsLoad to true.
10741076 /// This function should either return a nullptr, or a pointer to an IR-level
10751077 /// Instruction*. Even complex fence sequences can be represented by a
10761078 /// single Instruction* through an intrinsic to be lowered later.
1077 /// Backends with !getInsertFencesForAtomic() should keep a no-op here.
10781079 /// Backends should override this method to produce target-specific intrinsic
10791080 /// for their fences.
10801081 /// FIXME: Please note that the default implementation here in terms of
11001101 virtual Instruction *emitLeadingFence(IRBuilder<> &Builder,
11011102 AtomicOrdering Ord, bool IsStore,
11021103 bool IsLoad) const {
1103 if (!getInsertFencesForAtomic())
1104 return nullptr;
1105
11061104 if (isAtLeastRelease(Ord) && IsStore)
11071105 return Builder.CreateFence(Ord);
11081106 else
11121110 virtual Instruction *emitTrailingFence(IRBuilder<> &Builder,
11131111 AtomicOrdering Ord, bool IsStore,
11141112 bool IsLoad) const {
1115 if (!getInsertFencesForAtomic())
1116 return nullptr;
1117
11181113 if (isAtLeastAcquire(Ord))
11191114 return Builder.CreateFence(Ord);
11201115 else
14381433 /// Set the minimum stack alignment of an argument (in log2(bytes)).
14391434 void setMinStackArgumentAlignment(unsigned Align) {
14401435 MinStackArgumentAlignment = Align;
1441 }
1442
1443 /// Set if the DAG builder should automatically insert fences and reduce the
1444 /// order of atomic memory operations to Monotonic.
1445 void setInsertFencesForAtomic(bool fence) {
1446 InsertFencesForAtomic = fence;
14471436 }
14481437
14491438 public:
18551844 /// The preferred loop alignment.
18561845 unsigned PrefLoopAlignment;
18571846
1858 /// Whether the DAG builder should automatically insert fences and reduce
1859 /// ordering for atomics. (This will be set for for most architectures with
1860 /// weak memory ordering.)
1861 bool InsertFencesForAtomic;
18621847
18631848 /// If set to a physical register, this specifies the register that
18641849 /// llvm.savestack/llvm.restorestack should save and restore.
9999 assert((LI || SI || RMWI || CASI || isa(I)) &&
100100 "Unknown atomic instruction");
101101
102 if (TLI->getInsertFencesForAtomic()) {
102 if (TLI->shouldInsertFencesForAtomic(I)) {
103103 auto FenceOrdering = Monotonic;
104104 bool IsStore, IsLoad;
105105 if (LI && isAtLeastAcquire(LI->getOrdering())) {
513513 BasicBlock *BB = CI->getParent();
514514 Function *F = BB->getParent();
515515 LLVMContext &Ctx = F->getContext();
516 // If getInsertFencesForAtomic() returns true, then the target does not want
517 // to deal with memory orders, and emitLeading/TrailingFence should take care
518 // of everything. Otherwise, emitLeading/TrailingFence are no-op and we
516 // If shouldInsertFencesForAtomic() returns true, then the target does not
517 // want to deal with memory orders, and emitLeading/TrailingFence should take
518 // care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
519519 // should preserve the ordering.
520 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI);
520521 AtomicOrdering MemOpOrder =
521 TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder;
522 ShouldInsertFencesForAtomic ? Monotonic : SuccessOrder;
522523
523524 // In implementations which use a barrier to achieve release semantics, we can
524525 // delay emitting this barrier until we know a store is actually going to be
529530 // since in other cases the extra blocks naturally collapse down to the
530531 // minimal loop. Unfortunately, this puts too much stress on later
531532 // optimisations so we avoid emitting the extra logic in those cases too.
532 bool HasReleasedLoadBB = !CI->isWeak() && TLI->getInsertFencesForAtomic() &&
533 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
533534 SuccessOrder != Monotonic &&
534535 SuccessOrder != Acquire && !F->optForMinSize();
535536
600601 // the branch entirely.
601602 std::prev(BB->end())->eraseFromParent();
602603 Builder.SetInsertPoint(BB);
603 if (UseUnconditionalReleaseBarrier)
604 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
604605 TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
605606 /*IsLoad=*/true);
606607 Builder.CreateBr(StartBB);
616617 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
617618
618619 Builder.SetInsertPoint(ReleasingStoreBB);
619 if (!UseUnconditionalReleaseBarrier)
620 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
620621 TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
621622 /*IsLoad=*/true);
622623 Builder.CreateBr(TryStoreBB);
646647 // Make sure later instructions don't get reordered with a fence if
647648 // necessary.
648649 Builder.SetInsertPoint(SuccessBB);
649 TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true,
650 /*IsLoad=*/true);
650 if (ShouldInsertFencesForAtomic)
651 TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true,
652 /*IsLoad=*/true);
651653 Builder.CreateBr(ExitBB);
652654
653655 Builder.SetInsertPoint(NoStoreBB);
658660 Builder.CreateBr(FailureBB);
659661
660662 Builder.SetInsertPoint(FailureBB);
661 TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true,
662 /*IsLoad=*/true);
663 if (ShouldInsertFencesForAtomic)
664 TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true,
665 /*IsLoad=*/true);
663666 Builder.CreateBr(ExitBB);
664667
665668 // Finally, we have control-flow based knowledge of whether the cmpxchg
40344034 case ISD::ATOMIC_LOAD_UMAX:
40354035 case ISD::ATOMIC_CMP_SWAP: {
40364036 MVT VT = cast(Node)->getMemoryVT().getSimpleVT();
4037 RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
4037 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
40384038 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
40394039
40404040 std::pair Tmp = ExpandChainLibCall(LC, Node, false);
14031403 std::pair DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
14041404 unsigned Opc = Node->getOpcode();
14051405 MVT VT = cast(Node)->getMemoryVT().getSimpleVT();
1406 RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
1406 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
14071407 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
14081408
14091409 return ExpandChainLibCall(LC, Node, false);
666666 return UNKNOWN_LIBCALL;
667667 }
668668
669 RTLIB::Libcall RTLIB::getATOMIC(unsigned Opc, MVT VT) {
669 RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
670670 #define OP_TO_LIBCALL(Name, Enum) \
671671 case Name: \
672672 switch (VT.SimpleTy) { \
773773 PrefLoopAlignment = 0;
774774 GatherAllAliasesMaxDepth = 6;
775775 MinStackArgumentAlignment = 1;
776 InsertFencesForAtomic = false;
777776 MinimumJumpTableEntries = 4;
778777
779778 InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
839839 // the default expansion. If we are targeting a single threaded system,
840840 // then set them all for expand so we can lower them later into their
841841 // non-atomic form.
842 InsertFencesForAtomic = false;
842843 if (TM.Options.ThreadModel == ThreadModel::Single)
843844 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
844845 else if (Subtarget->hasAnyDataBarrier() && (!Subtarget->isThumb() ||
851852 // if they can be combined with nearby atomic loads and stores.
852853 if (!Subtarget->hasV8Ops()) {
853854 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
854 setInsertFencesForAtomic(true);
855 InsertFencesForAtomic = true;
855856 }
856857 } else {
857858 // If there's anything we can use as a barrier, go through custom lowering
1199611997 Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
1199711998 AtomicOrdering Ord, bool IsStore,
1199811999 bool IsLoad) const {
11999 if (!getInsertFencesForAtomic())
12000 return nullptr;
12001
1200212000 switch (Ord) {
1200312001 case NotAtomic:
1200412002 case Unordered:
1202412022 Instruction* ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
1202512023 AtomicOrdering Ord, bool IsStore,
1202612024 bool IsLoad) const {
12027 if (!getInsertFencesForAtomic())
12028 return nullptr;
12029
1203012025 switch (Ord) {
1203112026 case NotAtomic:
1203212027 case Unordered:
1207812073 bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
1207912074 AtomicCmpXchgInst *AI) const {
1208012075 return true;
12076 }
12077
12078 bool ARMTargetLowering::shouldInsertFencesForAtomic(
12079 const Instruction *I) const {
12080 return InsertFencesForAtomic;
1208112081 }
1208212082
1208312083 // This has so far only been implemented for MachO.
452452 bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
453453 unsigned Factor) const override;
454454
455 bool shouldInsertFencesForAtomic(const Instruction *I) const override;
455456 TargetLoweringBase::AtomicExpansionKind
456457 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
457458 bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
484485 /// ARMPCLabelIndex - Keep track of the number of ARM PC labels created.
485486 ///
486487 unsigned ARMPCLabelIndex;
488
489 // TODO: remove this, and have shouldInsertFencesForAtomic do the proper
490 // check.
491 bool InsertFencesForAtomic;
487492
488493 void addTypeForNEON(MVT VT, MVT PromotedLdStVT, MVT PromotedBitwiseVT);
489494 void addDRTypeForNEON(MVT VT);
17231723 setPrefLoopAlignment(4);
17241724 setPrefFunctionAlignment(4);
17251725 setMinFunctionAlignment(2);
1726 setInsertFencesForAtomic(false);
17271726 setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
17281727
17291728 if (EnableHexSDNodeSched)
395395 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
396396 }
397397
398 setInsertFencesForAtomic(true);
399398
400399 if (!Subtarget.hasMips32r2()) {
401400 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
560560 unsigned getJumpTableEncoding() const override;
561561 bool useSoftFloat() const override;
562562
563 bool shouldInsertFencesForAtomic(const Instruction *I) const override {
564 return true;
565 }
566
563567 /// Emit a sign-extension using sll/sra, seb, or seh appropriately.
564568 MachineBasicBlock *emitSignExtendToI32InReg(MachineInstr *MI,
565569 MachineBasicBlock *BB,
915915 break;
916916 }
917917
918 setInsertFencesForAtomic(true);
919918
920919 if (Subtarget.enableMachineScheduler())
921920 setSchedulingPreference(Sched::Source);
506506 unsigned Depth = 0) const override;
507507
508508 unsigned getPrefLoopAlignment(MachineLoop *ML) const override;
509
510 bool shouldInsertFencesForAtomic(const Instruction *I) const override {
511 return true;
512 }
509513
510514 Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
511515 bool IsStore, bool IsLoad) const override;
16021602 }
16031603
16041604 // ATOMICs.
1605 // FIXME: We insert fences for each atomics and generate sub-optimal code
1606 // for PSO/TSO. Also, implement other atomicrmw operations.
1607
1608 setInsertFencesForAtomic(true);
16091605
16101606 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Legal);
16111607 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32,
179179 return VT != MVT::f128;
180180 }
181181
182 bool shouldInsertFencesForAtomic(const Instruction *I) const override {
183 // FIXME: We insert fences for each atomics and generate
184 // sub-optimal code for PSO/TSO. (Approximately nobody uses any
185 // mode but TSO, which makes this even more silly)
186 return true;
187 }
188
182189 void ReplaceNodeResults(SDNode *N,
183190 SmallVectorImpl& Results,
184191 SelectionDAG &DAG) const override;
155155 // Atomic operations
156156 // We request a fence for ATOMIC_* instructions, to reduce them to Monotonic.
157157 // As we are always Sequential Consistent, an ATOMIC_FENCE becomes a no OP.
158 setInsertFencesForAtomic(true);
159158 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
160159 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
161160 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
228228 bool isVarArg,
229229 const SmallVectorImpl &ArgsFlags,
230230 LLVMContext &Context) const override;
231 bool shouldInsertFencesForAtomic(const Instruction *I) const override {
232 return true;
233 }
231234 };
232235 }
233236