llvm.org GIT mirror llvm / 1464846
Code generation for 'fence' instruction. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@136283 91177308-0d34-0410-b5e6-96231b3b80d8 Eli Friedman 9 years ago
16 changed file(s) with 119 addition(s) and 3 deletion(s). Raw diff Collapse all Expand all
590590 // operand specifying if the barrier applies to device and uncached memory
591591 // and produces an output chain.
592592 MEMBARRIER,
593
594 // OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope)
595 // This corresponds to the fence instruction. It takes an input chain, and
596 // two integer constants: an AtomicOrdering and a SynchronizationScope.
597 ATOMIC_FENCE,
593598
594599 // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
595600 // this corresponds to the atomic.lcs intrinsic.
203203 def SDTMemBarrier : SDTypeProfile<0, 5, [ // memory barier
204204 SDTCisSameAs<0,1>, SDTCisSameAs<0,2>, SDTCisSameAs<0,3>, SDTCisSameAs<0,4>,
205205 SDTCisInt<0>
206 ]>;
207 def SDTAtomicFence : SDTypeProfile<0, 2, [
208 SDTCisSameAs<0,1>, SDTCisPtrTy<0>
206209 ]>;
207210 def SDTAtomic3 : SDTypeProfile<1, 3, [
208211 SDTCisSameAs<0,2>, SDTCisSameAs<0,3>, SDTCisInt<0>, SDTCisPtrTy<1>
395398
396399 def membarrier : SDNode<"ISD::MEMBARRIER" , SDTMemBarrier,
397400 [SDNPHasChain, SDNPSideEffect]>;
401
402 def atomic_fence : SDNode<"ISD::ATOMIC_FENCE" , SDTAtomicFence,
403 [SDNPHasChain, SDNPSideEffect]>;
398404
399405 def atomic_cmp_swap : SDNode<"ISD::ATOMIC_CMP_SWAP" , SDTAtomic3,
400406 [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
29542954 Results.push_back(DAG.getConstant(0, MVT::i32));
29552955 Results.push_back(Node->getOperand(0));
29562956 break;
2957 case ISD::ATOMIC_FENCE:
29572958 case ISD::MEMBARRIER: {
29582959 // If the target didn't lower this, lower it to '__sync_synchronize()' call
2960 // FIXME: handle "fence singlethread" more efficiently.
29592961 TargetLowering::ArgListTy Args;
29602962 std::pair CallResult =
29612963 TLI.LowerCallTo(Node->getOperand(0), Type::getVoidTy(*DAG.getContext()),
57685768 #endif
57695769 case ISD::PREFETCH: return "Prefetch";
57705770 case ISD::MEMBARRIER: return "MemBarrier";
5771 case ISD::ATOMIC_FENCE: return "AtomicFence";
57715772 case ISD::ATOMIC_CMP_SWAP: return "AtomicCmpSwap";
57725773 case ISD::ATOMIC_SWAP: return "AtomicSwap";
57735774 case ISD::ATOMIC_LOAD_ADD: return "AtomicLoadAdd";
32203220 }
32213221
32223222 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
3223 llvm_unreachable("Not implemented yet");
3223 DebugLoc dl = getCurDebugLoc();
3224 SDValue Ops[3];
3225 Ops[0] = getRoot();
3226 Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
3227 Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
3228 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));
32243229 }
32253230
32263231 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
600600 // membarrier needs custom lowering; the rest are legal and handled
601601 // normally.
602602 setOperationAction(ISD::MEMBARRIER, MVT::Other, Custom);
603 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
603604 } else {
604605 // Set them all for expansion, which will force libcalls.
605606 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
607 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
606608 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, Expand);
607609 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, Expand);
608610 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand);
22762278 DAG.getConstant(DMBOpt, MVT::i32));
22772279 }
22782280
2281
2282 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
2283 const ARMSubtarget *Subtarget) {
2284 // FIXME: handle "fence singlethread" more efficiently.
2285 DebugLoc dl = Op.getDebugLoc();
2286 if (!Subtarget->hasDataBarrier()) {
2287 // Some ARMv6 cpus can support data barriers with an mcr instruction.
2288 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
2289 // here.
2290 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
2291 "Unexpected ISD::MEMBARRIER encountered. Should be libcall!");
2292 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
2293 DAG.getConstant(0, MVT::i32));
2294 }
2295
2296 AtomicOrdering FenceOrdering = static_cast(
2297 cast(Op.getOperand(1))->getZExtValue());
2298
2299 ARM_MB::MemBOpt DMBOpt;
2300 if (FenceOrdering == Release)
2301 DMBOpt = ARM_MB::ISHST;
2302 else
2303 DMBOpt = ARM_MB::ISH;
2304 return DAG.getNode(ARMISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0),
2305 DAG.getConstant(DMBOpt, MVT::i32));
2306 }
2307
22792308 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
22802309 const ARMSubtarget *Subtarget) {
22812310 // ARM pre v5TE and Thumb1 does not have preload instructions.
48334862 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
48344863 case ISD::VASTART: return LowerVASTART(Op, DAG);
48354864 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op, DAG, Subtarget);
4865 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
48364866 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
48374867 case ISD::SINT_TO_FP:
48384868 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
605605 (WMB)>;
606606 def : Pat<(membarrier (i64 imm), (i64 imm), (i64 imm), (i64 imm), (i64 imm)),
607607 (MB)>;
608
609 def : Pat<(atomic_fence (imm), (imm)), (MB)>;
608610
609611 //Basic Floating point ops
610612
9898
9999 // Blackfin has no intrinsics for these particular operations.
100100 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
101 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
101102 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
102103
103104 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
173173
174174 // SPU has no intrinsics for these particular operations:
175175 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
176 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
176177
177178 // SPU has no division/remainder instructions
178179 setOperationAction(ISD::SREM, MVT::i8, Expand);
876876 // Peepholes
877877 def : Pat<(store (i32 0), iaddr:$dst), (SWI (i32 R0), iaddr:$dst)>;
878878
879 // Atomic fence
880 def : Pat<(atomic_fence (imm), (imm)), (MEMBARRIER)>;
881
879882 //===----------------------------------------------------------------------===//
880883 // Floating Point Support
881884 //===----------------------------------------------------------------------===//
159159 // Use the default for now
160160 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
161161 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
162
162163 setOperationAction(ISD::MEMBARRIER, MVT::Other, Custom);
164 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
163165
164166 if (Subtarget->isSingleFloat())
165167 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
528530 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
529531 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
530532 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op, DAG);
533 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG);
531534 }
532535 return SDValue();
533536 }
15351538 DAG.getConstant(SType, MVT::i32));
15361539 }
15371540
1541 SDValue MipsTargetLowering::LowerATOMIC_FENCE(SDValue Op,
1542 SelectionDAG& DAG) const {
1543 // FIXME: Need pseudo-fence for 'singlethread' fences
1544 // FIXME: Set SType for weaker fences where supported/appropriate.
1545 unsigned SType = 0;
1546 DebugLoc dl = Op.getDebugLoc();
1547 return DAG.getNode(MipsISD::Sync, dl, MVT::Other, Op.getOperand(0),
1548 DAG.getConstant(SType, MVT::i32));
1549 }
1550
15381551 //===----------------------------------------------------------------------===//
15391552 // Calling Convention Implementation
15401553 //===----------------------------------------------------------------------===//
130130 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
131131 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
132132 SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG& DAG) const;
133 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const;
133134
134135 virtual SDValue
135136 LowerFormalArguments(SDValue Chain,
14711471 (i32 imm /*device*/)),
14721472 (SYNC)>;
14731473
1474 def : Pat<(atomic_fence (imm), (imm)), (SYNC)>;
1475
14741476 include "PPCInstrAltivec.td"
14751477 include "PPCInstr64Bit.td"
747747 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
748748 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
749749
750 // SPARC has no intrinsics for these particular operations.
750 // FIXME: There are instructions available for ATOMIC_FENCE
751 // on SparcV8 and later.
751752 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
753 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
752754
753755 setOperationAction(ISD::FSIN , MVT::f64, Expand);
754756 setOperationAction(ISD::FCOS , MVT::f64, Expand);
448448 if (Subtarget->hasXMM())
449449 setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
450450
451 // We may not have a libcall for MEMBARRIER so we should lower this.
452451 setOperationAction(ISD::MEMBARRIER , MVT::Other, Custom);
452 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
453453
454454 // On X86 and X86-64, atomic operations are lowered to locked instructions.
455455 // Locked instructions, in turn, have implicit fence semantics (all memory
94069406 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
94079407 }
94089408
9409 SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op,
9410 SelectionDAG &DAG) const {
9411 DebugLoc dl = Op.getDebugLoc();
9412 AtomicOrdering FenceOrdering = static_cast(
9413 cast(Op.getOperand(1))->getZExtValue());
9414 SynchronizationScope FenceScope = static_cast(
9415 cast(Op.getOperand(2))->getZExtValue());
9416
9417 // The only fence that needs an instruction is a sequentially-consistent
9418 // cross-thread fence.
9419 if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
9420 // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
9421 // no-sse2). There isn't any reason to disable it if the target processor
9422 // supports it.
9423 if (Subtarget->hasSSE2() || Subtarget->is64Bit())
9424 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
9425
9426 SDValue Chain = Op.getOperand(0);
9427 SDValue Zero = DAG.getConstant(0, MVT::i32);
9428 SDValue Ops[] = {
9429 DAG.getRegister(X86::ESP, MVT::i32), // Base
9430 DAG.getTargetConstant(1, MVT::i8), // Scale
9431 DAG.getRegister(0, MVT::i32), // Index
9432 DAG.getTargetConstant(0, MVT::i32), // Disp
9433 DAG.getRegister(0, MVT::i32), // Segment.
9434 Zero,
9435 Chain
9436 };
9437 SDNode *Res =
9438 DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops,
9439 array_lengthof(Ops));
9440 return SDValue(Res, 0);
9441 }
9442
9443 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
9444 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
9445 }
9446
9447
94099448 SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
94109449 EVT T = Op.getValueType();
94119450 DebugLoc DL = Op.getDebugLoc();
95259564 default: llvm_unreachable("Should not custom lower this!");
95269565 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG);
95279566 case ISD::MEMBARRIER: return LowerMEMBARRIER(Op,DAG);
9567 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op,DAG);
95289568 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op,DAG);
95299569 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
95309570 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
98489888 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
98499889 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
98509890 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
9891 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
98519892 }
98529893 }
98539894
828828 SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
829829 SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
830830 SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const;
831 SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
831832 SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
832833
833834 // Utility functions to help LowerVECTOR_SHUFFLE